def evaluate(self, pred_file, gold_file=""): sents, labels, pred_labels = collect_data_from_tsv(pred_file) p, r, f1, p_head, r_head, f1_head = self.f1_score(labels, pred_labels) return { "precision": "{:.2f}".format(p), "recall": "{:.2f}".format(r), "f1-score": "{:.2f}".format(f1), "precision-head": "{:.2f}".format(p_head), "recall-head": "{:.2f}".format(r_head), "f1-score-head": "{:.2f}".format(f1_head) }
def convert(self, tsvfile, **kwargs): if kwargs is not None: semfile = kwargs["semfile"] else: raise ("[!] semfile not found") if os.path.isfile(tsvfile) == False: raise ("[!] Data %s not found" % tsvfile) if os.path.isfile(semfile) == False: raise ("[!] Data %s not found" % semfile) # Collect sentences in tsv file sents, labels, pred_labels = collect_data_from_tsv(tsvfile) sent_idx = 0 conflict_sent = [ '912:1', '799:1', '1027:1', '2:1', '231:1', '463:26', '1008:1', '762:1', '786:1301', '786:1054', '416:1', '347:1', '11:1', '29:1', '32894246#870052#0', '33070309#423221#1', '32464601#418474#1', '11350539#680470#4', '11302357#835238#2', '35177381#521555#3', '11351451#805713#4', '11513049#499488#8', '33072753#1351349#2', '32936760#1397861#8', '32896473#439063#0', '32464601#418474#0', '11432442#650772#2', '11313290#1139539#1' ] # Collect sentences in semfile tree = ET.parse(semfile) root = tree.getroot() for sentence_tag in root.findall('sentence'): if sentence_tag.get("id") in conflict_sent: root.remove(sentence_tag) continue text = sentence_tag.find('text').text words, preds = sents[sent_idx], pred_labels[sent_idx] aspectterms = get_aspecterm(words, preds) aspectterms_tag = ET.Element("aspectTerms") for aspectterm in aspectterms: aspectterm_str_origin, from_idx, to_idx = self.gen_aspecterm_positions( aspectterm, sentence_tag.find('text').text) aspectterm_tag = ET.Element("aspectTerm") aspectterm_tag.set("term", aspectterm_str_origin) aspectterm_tag.set("from", str(from_idx)) aspectterm_tag.set("to", str(to_idx)) aspectterms_tag.append(aspectterm_tag) sentence_tag.append(aspectterms_tag) sent_idx += 1 fo_path = os.path.join(os.getcwd(), "data/evaluation/ate_pred.xml") tree.write(fo_path, encoding="utf-8") return fo_path
def evaluate(self, pred_file, gold_file=""): senti_gold = [] senti_pred = [] sents, labels, pred_labels = collect_data_from_tsv(pred_file) for sent, label, pred in zip(sents, labels, pred_labels): sub_senti_gold, sub_senti_pred = self.evaluate_sentence( sent, label, pred) senti_gold += sub_senti_gold senti_pred += sub_senti_pred senti_gold = np.array(senti_gold, dtype=np.int32) senti_pred = np.array(senti_pred, dtype=np.int32) accuracy = float(sum(senti_gold == senti_pred)) / len(senti_gold) return {"accuracy": "{0:.2f}".format(accuracy * 100)}
def convert(self, filename, **kwargs): sents, labels, pred_labels = collect_data_from_tsv(filename) fo_path = filename[:-4] + ".ATE.tsv" if kwargs is not None: if "fo_path" in kwargs: fo_path = kwargs["fo_path"] fo = open(fo_path, mode="w") for words, tags, preds in zip(sents, labels, pred_labels): for word, tag, pred in zip(words, tags, preds): fo.write("{0}\t{1}\t{2}\n".format(word, self.convert_label(tag), self.convert_label(pred))) fo.write("\n") fo.close() return fo_path
def evaluate(self, pred_file, gold_file=""): sentis_gold = [] sentis_pred = [] no_incorrect_ate = 0 no_correct_ate = 0 sents, labels, pred_labels = collect_data_from_tsv(pred_file) for sent, label, pred in zip(sents, labels, pred_labels): aspecterms, aspect_labels_gold, aspect_labels_pred = self.get_pred_infor( sent, label, pred) for aspectterm, aspect_label_gold, aspect_label_pred in zip( aspecterms, aspect_labels_gold, aspect_labels_pred): is_correct_ate = self.compare_2_lists(aspect_label_gold, aspect_label_pred) if is_correct_ate is False: no_incorrect_ate += 1 continue else: no_correct_ate += 1 senti_gold = self.BIO2Sentiment( [label.split("-")[-1] for label in aspect_label_gold]) senti_pred = self.BIO2Sentiment( [label.split("-")[-1] for label in aspect_label_pred]) sentis_gold.append(senti_gold) sentis_pred.append(senti_pred) sentis_gold = np.array(sentis_gold, dtype=np.int32) sentis_pred = np.array(sentis_pred, dtype=np.int32) accuracy = float(sum(sentis_gold == sentis_pred)) / len(sentis_gold) return { "accuracy": "{0:.2f}".format(accuracy * 100), "no_incorrect_ate": no_incorrect_ate, "no_correct_ate": no_correct_ate, "ate_correct_rate": "{0:.2f}".format((float(no_correct_ate) / (no_correct_ate + no_incorrect_ate)) * 100) }
def test_anago(keras_model_name="WCP", hand_features=None, task_name="ATEPC2", data_name="laptops"): DATA_ROOT = 'data' SAVE_ROOT = './models' # trained models LOG_ROOT = './logs' # checkpoint, tensorboard w_embedding_path = '/home/s1610434/Documents/Data/Vector/glove.twitter.27B.100d.txt' c_embedding_path = '/home/s1610434/Documents/Data/Vector/AmaYelp/GloVe/glove.char.100.txt' pos_embedding_path = '/home/s1610434/Documents/Data/Vector/AmaYelp/GloVe/glove.pos.100.txt' unipos_embedding_path = '/home/s1610434/Documents/Data/Vector/AmaYelp/GloVe/glove.unipos.100.txt' model_config = prepare_modelconfig(keras_model_name) training_config = TrainingConfig() training_config.max_epoch = 100 training_config.early_stopping = 20 print("-----{0}-----{1}-----{2}-----{3}-----".format( task_name, data_name, keras_model_name, hand_features)) save_path = SAVE_ROOT + "/{0}/{1}".format(data_name, task_name) train_path = os.path.join(DATA_ROOT, '{0}.{1}.train.tsv'.format(data_name, task_name)) test_path = os.path.join(DATA_ROOT, '{0}.{1}.test.tsv'.format(data_name, task_name)) train_dep_path = os.path.join( DATA_ROOT, '{0}.{1}.train.dep.tsv'.format(data_name, task_name)) test_dep_path = os.path.join( DATA_ROOT, '{0}.{1}.test.dep.tsv'.format(data_name, task_name)) # train set x_train_valid, y_train_valid, _ = collect_data_from_tsv(train_path) x_train_valid_dep = collect_dept_data_from_tsv(train_dep_path) # test set X_test, Y_test, _ = collect_data_from_tsv(test_path) X_test_dep = collect_dept_data_from_tsv(test_dep_path) # TODO Kfold split kf = KFold(n_splits=10) i_fold = 0 results = [] atepc_evaluator = ATEPCEvaluator() for train_index, valid_index in kf.split(x_train_valid): model_name = "{0}.{1}.{2}".format(keras_model_name, "{0}".format(hand_features), i_fold) X_train, X_valid = x_train_valid[train_index], x_train_valid[ valid_index] X_train_dep, X_valid_dep = x_train_valid_dep[ train_index], x_train_valid_dep[valid_index] Y_train, Y_valid = y_train_valid[train_index], y_train_valid[ valid_index] print("Data train: ", X_train.shape, Y_train.shape) print("Data valid: ", X_valid.shape, Y_valid.shape) print("Data test: ", X_test.shape, Y_test.shape) p = prepare_preprocessor(list(zip(X_train, X_train_dep)), Y_train, keras_model_name=keras_model_name, hand_features=hand_features) model_config.vocab_size = len(p.vocab_word) model_config.char_vocab_size = len(p.vocab_char) if keras_model_name.find("P") != -1: if hand_features is not None: if "UNIPOS" in hand_features: pos_embedding_path = unipos_embedding_path model_config.pos_vocab_size = len(p.pos_extractor.features_dict) if keras_model_name.find("H") != -1: # model_config.hand_feature_size = gen_no_hand_dimension(data_name, hand_features, keras_model_name) model_config.hand_feature_size = 53 print("model_config.hand_feature_size: ", str(model_config.hand_feature_size)) filepath = os.path.join(save_path, model_name) if os.path.isfile(filepath) is False: continue evaluator = anago.Evaluator(model_config, weights=model_name, save_path=save_path, preprocessor=p, keras_model_name=keras_model_name) print("--- Test phrase --- " + model_name) # print("Train ") # f1_score_train = evaluator.eval(list(zip(X_train, X_train_dep)), Y_train) # print("Validation ") # f1_score_valid = evaluator.eval(list(zip(X_valid, X_valid_dep)), Y_valid) # print("Test ") f1_score_test = evaluator.eval(list(zip(X_test, X_test_dep)), Y_test) print("---") i_fold += 1 # Kfold cross validation f_out_name = "data/{0}.{1}.test.pred.tsv".format(data_name, task_name) f_out = open(f_out_name, "w") ## Tagging tagger = anago.Tagger(model_config, model_name, save_path=save_path, preprocessor=p, keras_model_name=keras_model_name) for x, y in zip(list(zip(X_test, X_test_dep)), Y_test): result = tagger.predict(x) for word, label, pred in zip(x[0], y, result): f_out.write("{0}\t{1}\t{2}\n".format(word, label, pred)) f_out.write("\n") f_out.close() ate_f1, apc_acc, c_apc_acc = atepc_evaluator.evaluate(f_out_name) results.append([ate_f1, apc_acc, c_apc_acc]) print(results[-1]) print("-----All-----{0}--{1}".format(keras_model_name, data_name)) for result in results: print(result) print("-----AVG-----") results_np = np.array(results, dtype=np.float32) print(results_np.mean(axis=0)) print("-------------")
def evaluate(self, pred_file, gold_file=""): sents, labels, pred_labels = collect_data_from_tsv(pred_file) return self._evaluate(labels, pred_labels)
} print("-----{0}-----{1}-----{2}-----{3}-----".format( task_name, data_name, keras_model_name, hand_features)) save_path = SAVE_ROOT + "/{0}/{1}".format(data_name, task_name) train_path = os.path.join(DATA_ROOT, '{0}.{1}.train.tsv'.format(data_name, task_name)) test_path = os.path.join(DATA_ROOT, '{0}.{1}.test.tsv'.format(data_name, task_name)) train_dep_path = os.path.join( DATA_ROOT, '{0}.{1}.train.dep.tsv'.format(data_name, task_name)) test_dep_path = os.path.join( DATA_ROOT, '{0}.{1}.test.dep.tsv'.format(data_name, task_name)) # train set x_train_valid, y_train_valid, _ = collect_data_from_tsv(train_path) x_train_valid_dep = collect_dept_data_from_tsv(train_dep_path) # test set X_test, Y_test, _ = collect_data_from_tsv(test_path) X_test_dep = collect_dept_data_from_tsv(test_dep_path) # TODO Kfold split kf = KFold(n_splits=10) i_fold = 0 pos_fe = POSExtractor() print(len(pos_fe.features_dict)) count_train = 0 count_valid = 0