def test(self): #train - data segments pre_train = self.data_model.preceding mid_train = self.data_model.middle suc_train = self.data_model.succeeding c1_train = self.data_model.concept1 c2_train = self.data_model.concept2 y_train = self.data_model.train_label binary_y_train = self.data_model.binarize_labels(y_train, True) # test data segments pre_test = self.data_model.test_preceding mid_test = self.data_model.test_middle suc_test = self.data_model.test_succeeding c1_test = self.data_model.test_concept1 c2_test = self.data_model.test_concept2 track_test = self.data_model.test_track if not self.data_model.write_Predictions: y_test = self.data_model.y_test binary_y_test = self.data_model.binarize_labels(y_test, True) labels = [str(i) for i in self.data_model.encoder.classes_] cv_model = self.build_segment_cnn(len( self.data_model.encoder.classes_)) cv_model.fit([pre_train, mid_train, suc_train, c1_train, c2_train], binary_y_train, epochs=self.epochs, batch_size=self.batch_size) if self.data_model.write_Predictions: pred = evaluate.predict_test_only( cv_model, [pre_test, mid_test, suc_test, c1_test, c2_test], labels) # save files in numpy to write predictions in BRAT format np.save('track', np.array(track_test)) np.save('pred', np.array(pred)) Predictions(self.initial_predictions, self.final_predictions, self.write_No_rel) else: y_pred, y_true = evaluate.predict( cv_model, [pre_test, mid_test, suc_test, c1_test, c2_test], binary_y_test, labels) print(classification_report(y_true, y_pred, labels=labels)) print(confusion_matrix(y_true, y_pred))
def test(self): """ Train - Test - Split """ x_train = self.data_model.train y_train = self.data_model.train_label binary_y_train = self.data_model.binarize_labels(y_train, True) labels = [str(i) for i in self.data_model.encoder.classes_] x_test = self.data_model.x_test track_test = self.data_model.test_track if not self.data_model.write_Predictions: y_test = self.data_model.y_test binary_y_test = self.data_model.binarize_labels(y_test, True) cv_model = self.model_without_Label( len(self.data_model.encoder.classes_)) if self.data_model.multilabel: cv_model.fit(x_train, binary_y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.multilabel_predict( cv_model, x_test, binary_y_test) print(classification_report(y_true, y_pred, target_names=labels)) else: cv_model = self.fit_Model(cv_model, x_train, binary_y_train) if self.data_model.write_Predictions: pred = evaluate.predict_test_only(cv_model, x_test, labels) # save files in numpy to write predictions in BRAT format np.save('track', np.array(track_test)) np.save('pred', np.array(pred)) Predictions(self.initial_predictions, self.final_predictions, self.write_No_rel) else: y_pred, y_true = evaluate.predict(cv_model, x_test, binary_y_test, labels) print(confusion_matrix(y_true, y_pred)) print( classification_report(y_true, y_pred, target_names=labels))
def cross_validate(self, num_folds=5): """ Train the NN model while running cross validation. :param num_folds: no of CV fold (default = 5) """ X_data = self.data_model.train Y_data = self.data_model.train_label if num_folds <= 1: raise ValueError( "Number of folds for cross validation must be greater than 1") assert X_data is not None and Y_data is not None, \ "Must have features and labels extracted for cross validation" skf = StratifiedKFold(n_splits=num_folds, shuffle=True) skf.get_n_splits(X_data, Y_data) evaluation_statistics = {} fold = 1 for train_index, test_index in skf.split(X_data, Y_data): binary_Y = self.data_model.binarize_labels(Y_data, True) x_train, x_test = X_data[train_index], X_data[test_index] y_train, y_test = binary_Y[train_index], binary_Y[test_index] print("Training Fold %i", fold) labels = [str(i) for i in self.data_model.encoder.classes_] cv_model = self.define_model(len(self.data_model.encoder.classes_)) cv_model, loss, acc = self.fit_Model(cv_model, x_train, y_train) y_pred, y_true = evaluate.predict(cv_model, x_test, y_test, labels) fold_statistics = evaluate.cv_evaluation_fold( y_pred, y_true, labels) evaluation_statistics[fold] = fold_statistics fold += 1 evaluate.cv_evaluation(labels, evaluation_statistics)
def cross_validate(self, num_folds=5): """ Train the CNN model while running cross validation. :param num_folds: no of fold in CV (default = 5) """ X_data = self.data_model.train Y_data = self.data_model.train_label if self.data_model.with_Labels: C1_label = self.data_model.train_concept1_label C2_label = self.data_model.train_concept1_label if num_folds <= 1: raise ValueError( "Number of folds for cross validation must be greater than 1") assert X_data is not None and Y_data is not None, \ "Must have features and labels extracted for cross validation" skf = StratifiedKFold(n_splits=num_folds, shuffle=True) skf.get_n_splits(X_data, Y_data) evaluation_statistics = {} fold = 1 originalclass = [] predictedclass = [] #to track the entity pairs for each relation brat_track = [] if self.data_model.multilabel: binary_Y = self.data_model.binarize_labels(Y_data, True) for train_index, test_index in skf.split(X_data, binary_Y.argmax(1)): x_train, x_test = X_data[train_index], X_data[test_index] y_train, y_test = binary_Y[train_index], binary_Y[test_index] print("Training Fold %i", fold) labels = [str(i) for i in self.data_model.encoder.classes_] cv_model = self.model_without_Label( len(self.data_model.encoder.classes_)) cv_model.fit(x_train, y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.multilabel_predict( cv_model, x_test, y_test) originalclass.extend(y_true) predictedclass.extend(y_pred) print( "--------------------------- Results ------------------------------------" ) print( classification_report(y_true, y_pred, target_names=labels)) print( "--------------------- Results --------------------------------" ) print( classification_report(np.array(originalclass), np.array(predictedclass), target_names=labels)) else: Track = self.data_model.train_track for train_index, test_index in skf.split(X_data, Y_data): binary_Y = self.data_model.binarize_labels(Y_data, True) y_train, y_test = binary_Y[train_index], binary_Y[test_index] x_train, x_test = X_data[train_index], X_data[test_index] train_track, test_track = Track[train_index], Track[test_index] labels = [str(i) for i in self.data_model.encoder.classes_] if self.data_model.with_Labels: binary_C1_label = self.data_model.binarize_labels( C1_label, True) binary_C2_label = self.data_model.binarize_labels( C2_label, True) if self.data_model.generalize: train_C1_label, test_C1_label = binary_C1_label[ train_index], binary_C1_label[test_index] train_C2_label, test_C2_label = binary_C2_label[ train_index], binary_C2_label[test_index] cv_model = self.model_with_Label( len(self.data_model.encoder.classes_), train_C1_label, train_C2_label) cv_model.fit([x_train, train_C1_label, train_C2_label], y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.predict( cv_model, [x_test, test_C1_label, test_C2_label], y_test, labels) else: train_C1_label, test_C1_label = binary_C1_label[ train_index], binary_C1_label[test_index] cv_model = self.model_with_Label( len(self.data_model.encoder.classes_), train_C1_label) cv_model.fit([x_train, train_C1_label], y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.predict( cv_model, [x_test, test_C1_label], y_test, labels) print(y_true, y_pred) else: cv_model = self.model_without_Label( len(self.data_model.encoder.classes_)) cv_model = self.fit_Model(cv_model, x_train, y_train) y_pred, y_true = evaluate.predict(cv_model, x_test, y_test, labels) originalclass.extend(y_true) predictedclass.extend(y_pred) brat_track.extend(test_track) print( "--------------------------- Results ------------------------------------" ) print(classification_report(y_true, y_pred, labels=labels)) # print(confusion_matrix(y_true, y_pred)) fold_statistics = evaluate.cv_evaluation_fold( y_pred, y_true, labels) evaluation_statistics[fold] = fold_statistics fold += 1 print( "--------------------- Results --------------------------------" ) print( classification_report(np.array(originalclass), np.array(predictedclass), target_names=labels)) print( confusion_matrix(np.array(originalclass), np.array(predictedclass))) if self.data_model.write_Predictions: # save files in numpy to write predictions in BRAT format np.save('track', np.array(brat_track)) np.save('pred', np.array(predictedclass)) Predictions(self.initial_predictions, self.final_predictions, self.write_No_rel) #print results using MedaCy evaluation (similar to the sklearn evaluation above) print( "---------------------medacy Results --------------------------------" ) evaluate.cv_evaluation(labels, evaluation_statistics)
def end_to_end_test(self): if self.data_model.multilabel: x_train = self.data_model.true_train_x y_train = self.data_model.true_train_y binary_y_train = self.data_model.binarize_labels(y_train, True) x_test = self.data_model.true_test_x y_test = self.data_model.true_test_y binary_y_test = self.data_model.binarize_labels(y_test, True) labels = ['no', 'yes'] else: x_train = self.data_model.train y_train = self.data_model.train_label binary_y_train = self.data_model.binarize_labels(y_train, True) x_test = self.data_model.x_test y_test = self.data_model.y_test binary_y_test = self.data_model.binarize_labels(y_test, True) labels = [str(i) for i in self.data_model.encoder.classes_] cv_model = self.model_without_Label(len(labels)) cv_model = self.fit_Model(cv_model, x_train, binary_y_train) y_pred, y_true = evaluate.predict(cv_model, x_test, binary_y_test, labels) print( "--------------------- binary results ---------------------------------" ) print(confusion_matrix(y_true, y_pred)) print(classification_report(y_true, y_pred, target_names=labels)) x_train1, y_train1, x_test1, y_test1 = self.data_model.remove_instances( y_pred) if self.data_model.multilabel: self.data_model.binary_label = False df_train = model.reduce_duplicate_data(x_train1, y_train1) y_train2 = df_train.label.tolist() y_train1 = y_train2 df_test = model.reduce_duplicate_data(x_test1, y_test1) y_test2 = df_test.label.tolist() y_test1 = y_test2 train, test, word_index = self.data_model.vectorize_words( df_train.sentence, df_test.sentence) x_train1 = train x_test1 = test binary_y_train1 = self.data_model.binarize_labels(y_train1, True) labels1 = [str(i) for i in self.data_model.encoder.classes_] binary_y_test1 = self.data_model.binarize_labels(y_test1, True) cv_model1 = self.model_without_Label(len(labels1)) if self.data_model.multilabel: cv_model1.fit(x_train1, binary_y_train1, epochs=self.epochs, batch_size=self.batch_size) y_pred1, y_true1 = evaluate.multilabel_predict( cv_model1, x_test1, binary_y_test1) else: cv_model1 = self.fit_Model(cv_model1, np.array(x_train1), np.array(binary_y_train1)) y_pred1, y_true1 = evaluate.predict(cv_model1, np.array(x_test1), np.array(binary_y_test1), labels1) print( "--------------------- Final results ---------------------------------" ) print(confusion_matrix(y_true1, y_pred1)) print(classification_report(y_true1, y_pred1, target_names=labels1))
def cross_validate(self, num_folds=5): """ Perform cross validation :param num_folds: no of fold for cross validation (default = 5) """ Pre_data = self.data_model.preceding Mid_data = self.data_model.middle Suc_data = self.data_model.succeeding C1_data = self.data_model.concept1 C2_data = self.data_model.concept2 Track = self.data_model.train_track Y_data = self.data_model.train_label if num_folds <= 1: raise ValueError( "Number of folds for cross validation must be greater than 1") skf = StratifiedKFold(n_splits=num_folds, shuffle=True) skf.get_n_splits(C1_data, Y_data) evaluation_statistics = {} fold = 1 originalclass = [] predictedclass = [] # to track the entity pairs for each relation brat_track = [] for train_index, test_index in skf.split(C1_data, Y_data): binary_Y = self.data_model.binarize_labels(Y_data, True) pre_train, pre_test = Pre_data[train_index], Pre_data[test_index] mid_train, mid_test = Mid_data[train_index], Mid_data[test_index] suc_train, suc_test = Suc_data[train_index], Suc_data[test_index] c1_train, c1_test = C1_data[train_index], C1_data[test_index] c2_train, c2_test = C2_data[train_index], C2_data[test_index] track_train, track_test = Track[train_index], Track[test_index] y_train, y_test = binary_Y[train_index], binary_Y[test_index] labels = [str(i) for i in self.data_model.encoder.classes_] cv_model = self.build_segment_cnn( len(self.data_model.encoder.classes_)) cv_model.fit([pre_train, mid_train, suc_train, c1_train, c2_train], y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.predict( cv_model, [pre_test, mid_test, suc_test, c1_test, c2_test], y_test, labels) fold_statistics = evaluate.cv_evaluation_fold( y_pred, y_true, labels) originalclass.extend(y_true) predictedclass.extend(y_pred) brat_track.extend(track_test) print( "--------------------------- Results ------------------------------------" ) print(classification_report(y_true, y_pred, labels=labels)) # print(confusion_matrix(y_true, y_pred)) evaluation_statistics[fold] = fold_statistics fold += 1 if self.data_model.write_Predictions: # save files in numpy to write predictions in BRAT format np.save('track', np.array(brat_track)) np.save('pred', np.array(predictedclass)) Predictions(self.initial_predictions, self.final_predictions, self.write_No_rel) print("--------------------- Results --------------------------------") print( classification_report(np.array(originalclass), np.array(predictedclass), target_names=labels)) print( confusion_matrix(np.array(originalclass), np.array(predictedclass))) # print results using MedaCy evaluation (similar to the sklearn evaluation above) print( "---------------------medacy Results --------------------------------" ) evaluate.cv_evaluation(labels, evaluation_statistics)
def end_to_end_test(self): pre_train = self.data_model.preceding mid_train = self.data_model.middle suc_train = self.data_model.succeeding c1_train = self.data_model.concept1 c2_train = self.data_model.concept2 y_train = self.data_model.train_label binary_y_train = self.data_model.binarize_labels(y_train, True) labels = [str(i) for i in self.data_model.encoder.classes_] pre_test = self.data_model.test_preceding mid_test = self.data_model.test_middle suc_test = self.data_model.test_succeeding c1_test = self.data_model.test_concept1 c2_test = self.data_model.test_concept2 y_test = self.data_model.y_test binary_y_test = self.data_model.binarize_labels(y_test, True) cv_model = self.build_segment_cnn(len( self.data_model.encoder.classes_)) cv_model.fit([pre_train, mid_train, suc_train, c1_train, c2_train], binary_y_train, epochs=self.epochs, batch_size=self.batch_size) y_pred, y_true = evaluate.predict( cv_model, [pre_test, mid_test, suc_test, c1_test, c2_test], binary_y_test, labels) print( "--------------------- binary results ---------------------------------" ) print(confusion_matrix(y_true, y_pred)) print(classification_report(y_true, y_pred, target_names=labels)) df_train, df_test = self.data_model.remove_instances(y_pred) binary_y_train1 = self.data_model.binarize_labels( df_train.label.tolist(), True) labels1 = [str(i) for i in self.data_model.encoder.classes_] binary_y_test1 = self.data_model.binarize_labels( df_test.true.tolist(), True) cv_model1 = self.build_segment_cnn(len(labels1)) cv_model1.fit([ df_train.preceding.tolist(), df_train.middle.tolist(), df_train.succeeding.tolist(), df_train.c1.tolist(), df_train.c2.tolist() ], binary_y_train1, epochs=self.epochs, batch_size=self.batch_size) y_pred1, y_true1 = evaluate.predict(cv_model1, [ df_test.preceding.tolist(), df_test.middle.tolist(), df_test.succeeding.tolist(), df_test.c1.tolist(), df_test.c2.tolist() ], binary_y_test1, labels1) print( "--------------------- Final results ---------------------------------" ) print(confusion_matrix(y_true1, y_pred1)) print(classification_report(y_true1, y_pred1, target_names=labels1))