def draw_roc_curve_saved_model(self): self.logger.println("drawing roc curve from saved model") start_time = timeit.default_timer() cs = CrfSuite() crf = cs.load_model("current_crf_model.pkl") dataset = Dataset() data = dataset.read(nr_of_files=1000) nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data) data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data) data = data1 + data2 data = dataset.shuffle_data(data) train_set, test_set = dataset.split_dataset(data) we_model = WeModel() w2v_model = we_model.read() we_model = None word2count, word2idx = dataset.encode_dataset(train_set) f_generator = FeatureGenerator(w2v_model, word2count, word2idx) w2v_model = None train_features = f_generator.generate_features_docs(train_set) y_train = f_generator.generate_true_outcome(train_set) test_features = f_generator.generate_features_docs(test_set) y_test = f_generator.generate_true_outcome(test_set) f_generator = None evaluator = Evaluator() evaluator.draw_roc_proba(crf, test_features, y_test)
def train_model(self, nr_of_files=-1): self.logger.println("train model called") start_time = timeit.default_timer() cs = CrfSuite() dataset = Dataset() data = dataset.read(nr_of_files=nr_of_files) nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data) data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data) data = data1 + data2 data = dataset.shuffle_data(data) train_set, test_set = dataset.split_dataset(data) we_model = WeModel() w2v_model = we_model.train( data) # optionally load a pretrained model here we_model.save(w2v_model) we_model = None word2count, word2idx = dataset.encode_dataset(train_set) f_generator = FeatureGenerator(w2v_model, word2count, word2idx) w2v_model = None train_features = f_generator.generate_features_docs(train_set) y_train = f_generator.generate_true_outcome(train_set) test_features = f_generator.generate_features_docs(test_set) y_test = f_generator.generate_true_outcome(test_set) f_generator = None model = cs.train_model(train_features, y_train) cs.save_model(model) y_train_pred = cs.test_model(model, train_features) y_test_pred = cs.test_model(model, test_features) print("printing training results") cs.print_classification_report(dataset.docs2lines(y_train), y_train_pred) score_train = cs.score_model(dataset.docs2lines(y_train), y_train_pred) print("training f1 score: %s" % score_train) print("printing test results") cs.print_classification_report(dataset.docs2lines(y_test), y_test_pred) score_test = cs.score_model(dataset.docs2lines(y_test), y_test_pred) print("test f1 score: %s" % score_test) elapsed_seconds = timeit.default_timer() - start_time self.logger.print_time_taken("train model operation took", elapsed_seconds) evaluator = Evaluator() evaluator.perform_roc_analysis(dataset.docs2lines(y_train), y_train_pred) evaluator.perform_roc_analysis(dataset.docs2lines(y_test), y_test_pred)
def optimise_model(self, argv): self.logger.println("optimise model called") start_time = timeit.default_timer() cs = CrfSuite() dataset = Dataset() data = dataset.read(nr_of_files=argv) we_model = WeModel() w2v_model = we_model.train( data) # optionally load a pretrained model here we_model.save(w2v_model) word2count, word2idx = dataset.encode_dataset(data) f_generator = FeatureGenerator(w2v_model, word2count, word2idx) train_features = f_generator.generate_features_docs(data) y_train = f_generator.generate_true_outcome(data) cs.optimise_model(train_features, y_train) elapsed_seconds = timeit.default_timer() - start_time self.logger.print_time_taken("optimise model operation took", elapsed_seconds)
def train_model_learning_curve(self, arg): self.logger.println("train model called") start_time = timeit.default_timer() cs = CrfSuite() dataset = Dataset() data = dataset.read(nr_of_files=arg) nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data) data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data) data = data1 + data2 data = dataset.shuffle_data(data) train_set, test_set = dataset.split_dataset(data) we_model = WeModel() w2v_model = we_model.train( data) # optionally load a pretrained model here #w2v_model = we_model.load_pretrained_model() # optionally load a pretrained model here word2count, word2idx = dataset.encode_dataset(train_set) f_generator = FeatureGenerator(w2v_model, word2count, word2idx) train_features = f_generator.generate_features_docs(train_set) y_train = f_generator.generate_true_outcome(train_set) cs.plot_learning_curve(train_features, y_train) plt.show() elapsed_seconds = timeit.default_timer() - start_time self.logger.print_time_taken("train model operation took", elapsed_seconds)
def get_feature_generator_results(self): input = [[("this", "", "", ""), ("is", "", "", ""), ("a", "", "", ""), ("test", "", "", "")], [("this", "", "", ""), ("is", "", "", ""), ("a", "", "", ""), ("test", "", "", "")]] dataset = Dataset() word2count, word2idx = dataset.encode_dataset([input]) f_generator = FeatureGenerator(self.w2v_model, word2count, word2idx) X = f_generator.generate_features_docs([input]) y = f_generator.generate_true_outcome([input]) # run tests on X and y return input, X, y
def perform_bootstrapping(self, dataset, sample_size, iterations): """ bootstraps a sample n times. Averages the precision, recall, f1, tpr and fpr for each of the entities. Prints results of precision, recall and f1. Plots roc curves for tpr and fpr of each entity. """ training_scores = [] test_scores = [] emp_pos_scores = np.empty(shape=(0, 3), dtype='float64') emp_comp_scores = np.empty(shape=(0, 3), dtype='float64') edu_major_scores = np.empty(shape=(0, 3), dtype='float64') edu_inst_scores = np.empty(shape=(0, 3), dtype='float64') mean_fpr = np.linspace(0, 1, 100) lb = LabelBinarizer() emp_pos_tpr = np.empty(shape=(0, 3), dtype='float64') emp_pos_fpr = np.empty(shape=(0, 3), dtype='float64') emp_comp_tpr = np.empty(shape=(0, 3), dtype='float64') emp_comp_fpr = np.empty(shape=(0, 3), dtype='float64') edu_major_tpr = np.empty(shape=(0, 3), dtype='float64') edu_major_fpr = np.empty(shape=(0, 3), dtype='float64') edu_inst_tpr = np.empty(shape=(0, 3), dtype='float64') edu_inst_fpr = np.empty(shape=(0, 3), dtype='float64') for x in range(0, iterations): print("iteration nr %s" % x) sampled_train_set, oob_test_set = self.resample_data( dataset, sample_size, return_leftovers=True) cs = CrfSuite() ds = Dataset() we_model = WeModel() w2v_model = we_model.train( dataset) # optionally load a pretrained model here word2count, word2idx = ds.encode_dataset(sampled_train_set) f_generator = FeatureGenerator(w2v_model, word2count, word2idx) train_features = f_generator.generate_features_docs( sampled_train_set) y_train = f_generator.generate_true_outcome(sampled_train_set) test_features = f_generator.generate_features_docs(oob_test_set) y_test = f_generator.generate_true_outcome(oob_test_set) trainer = cs.train_model(train_features, y_train) y_train_pred = cs.test_model(trainer, train_features) y_test_pred = cs.test_model(trainer, test_features) score_train = cs.score_model(ds.docs2lines(y_train), y_train_pred) score_test = cs.score_model(ds.docs2lines(y_test), y_test_pred) y_true_combined = lb.fit_transform( list(chain.from_iterable(ds.docs2lines(y_test)))) y_pred_combined = lb.transform( list(chain.from_iterable(y_test_pred))) class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)} """ # fpr and tpr for one class temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EMP-POS"]], y_pred_combined[:, class_indices["B-EMP-POS"]], pos_label=1) temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EMP-POS"]], y_pred_combined[:, class_indices["I-EMP-POS"]], pos_label=1) temp_fpr = np.vstack([temp_fpr, temp_fpr1]) temp_tpr = np.vstack([temp_tpr, temp_tpr1]) emp_pos_tpr = np.vstack([emp_pos_tpr, temp_tpr.mean(axis=0)]) emp_pos_fpr = np.vstack([emp_pos_fpr, temp_fpr.mean(axis=0)]) temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64') temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EMP-COMP"]], y_pred_combined[:, class_indices["B-EMP-COMP"]], pos_label=1) temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EMP-COMP"]], y_pred_combined[:, class_indices["I-EMP-COMP"]], pos_label=1) temp_fpr = np.vstack([temp_fpr, temp_fpr1]) temp_tpr = np.vstack([temp_tpr, temp_tpr1]) emp_comp_tpr = np.vstack([emp_comp_tpr, temp_tpr.mean(axis=0)]) emp_comp_fpr = np.vstack([emp_comp_fpr, temp_fpr.mean(axis=0)]) temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64') temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EDU-MAJOR"]], y_pred_combined[:, class_indices["B-EDU-MAJOR"]], pos_label=1) temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EDU-MAJOR"]], y_pred_combined[:, class_indices["I-EDU-MAJOR"]], pos_label=1) temp_fpr = np.vstack([temp_fpr, temp_fpr1]) temp_tpr = np.vstack([temp_tpr, temp_tpr1]) edu_major_tpr = np.vstack([edu_major_tpr, temp_tpr.mean(axis=0)]) edu_major_fpr = np.vstack([edu_major_fpr, temp_fpr.mean(axis=0)]) temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64') temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EDU-INST"]], y_pred_combined[:, class_indices["B-EDU-INST"]], pos_label=1) temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EDU-INST"]], y_pred_combined[:, class_indices["I-EDU-INST"]], pos_label=1) temp_fpr = np.vstack([temp_fpr, temp_fpr1]) temp_tpr = np.vstack([temp_tpr, temp_tpr1]) edu_inst_tpr = np.vstack([edu_inst_tpr, temp_tpr.mean(axis=0)]) edu_inst_fpr = np.vstack([edu_inst_fpr, temp_fpr.mean(axis=0)]) """ emp_pos_scores = np.vstack([ emp_pos_scores, self.entity_scorer(ds.docs2lines(y_test), y_test_pred, "EMP-POS") ]) emp_comp_scores = np.vstack([ emp_comp_scores, self.entity_scorer(ds.docs2lines(y_test), y_test_pred, "EMP-COMP") ]) edu_major_scores = np.vstack([ edu_major_scores, self.entity_scorer(ds.docs2lines(y_test), y_test_pred, "EDU-MAJOR") ]) edu_inst_scores = np.vstack([ edu_inst_scores, self.entity_scorer(ds.docs2lines(y_test), y_test_pred, "EDU-INST") ]) w2v_model = None train_features = test_features = None """ print("EMP-POS") print("precision %s" % np.mean(emp_pos_scores[:,0])) print("recall %s" % np.mean(emp_pos_scores[:,1])) print("f1 %s" % np.mean(emp_pos_scores[:,2])) print("EMP-COMP") print("precision %s" % np.mean(emp_comp_scores[:,0])) print("recall %s" % np.mean(emp_comp_scores[:,1])) print("f1 %s" % np.mean(emp_comp_scores[:,2])) print("EDU-MAJOR") print("precision %s" % np.mean(edu_major_scores[:,0])) print("recall %s" % np.mean(edu_major_scores[:,1])) print("f1 %s" % np.mean(edu_major_scores[:,2])) print("EDU-INST") print("precision %s" % np.mean(edu_inst_scores[:,0])) print("recall %s" % np.mean(edu_inst_scores[:,1])) print("f1 %s" % np.mean(edu_inst_scores[:,2])) emp_pos_tpr = emp_pos_tpr.mean(axis=0) emp_pos_fpr = emp_pos_fpr.mean(axis=0) emp_comp_tpr = emp_comp_tpr.mean(axis=0) emp_comp_fpr = emp_comp_fpr.mean(axis=0) edu_major_tpr = edu_major_tpr.mean(axis=0) edu_major_fpr = edu_major_fpr.mean(axis=0) edu_inst_tpr = edu_inst_tpr.mean(axis=0) edu_inst_fpr = edu_inst_fpr.mean(axis=0) lw=2 plt.subplot(221) plt.plot(emp_pos_fpr, emp_pos_tpr, color='g', linestyle='--', label='EMP-POS', lw=lw) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC') plt.legend(loc="lower right") plt.subplot(222) plt.plot(emp_comp_fpr, emp_comp_tpr, color='g', linestyle='--', label='EMP-COMP', lw=lw) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC') plt.legend(loc="lower right") plt.subplot(223) plt.plot(edu_major_fpr, edu_major_tpr, color='g', linestyle='--', label='EDU-MAJOR', lw=lw) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC') plt.legend(loc="lower right") plt.subplot(224) plt.plot(edu_inst_fpr, edu_inst_tpr, color='g', linestyle='--', label='EDU-INST', lw=lw) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC') plt.legend(loc="lower right") plt.show() """ return emp_pos_scores, emp_comp_scores, edu_inst_scores, edu_major_scores