def evaluate(eval_dic): #evaluates the outputs against gold files using conll18_ud_eval module by UD #eval_dic should be a dictionary with gold filenames as keys and system filenames as values import conll18_ud_eval table = [] for k in eval_dic: gfile = open(k, 'r', encoding='utf-8') sfile = open(eval_dic[k], 'r', encoding='utf-8') g = conll18_ud_eval.load_conllu(gfile) s = conll18_ud_eval.load_conllu(sfile) gfile.close() sfile.close() evaluation = conll18_ud_eval.evaluate(g, s) newdic = {} name = k.split('\\')[-1] name = name.replace('-ud-', '_') name = name.replace('.conllu', '') newdic['name'] = name for component in [ 'Sentences', 'Tokens', 'Words', 'Lemmas', 'UPOS', 'XPOS', 'UFeats', 'UAS', 'LAS' ]: newdic[component + ' precision'] = evaluation[component].precision newdic[component + ' recall'] = evaluation[component].recall newdic[component + ' f1'] = evaluation[component].f1 newdic[ component + ' aligned_accuracy'] = evaluation[component].aligned_accuracy newdic['tokens'] = evaluation['Tokens'].gold_total newdic['words'] = evaluation['Words'].gold_total newdic['sentences'] = evaluation['Sentences'].gold_total table.append(newdic) df = pandas.DataFrame(table) return df
def evaluate(self, dataset_name, dataset, dataset_conllu, args): import io conllu, sentences = io.StringIO(), 0 self.session.run(self.reset_metrics) while not dataset.epoch_finished(): sentence_lens, word_ids, charseq_ids, charseqs, charseq_lens = dataset.next_batch(args.batch_size) feeds = {self.is_training: False, self.sentence_lens: sentence_lens, self.charseqs: charseqs[train.FORMS], self.charseq_lens: charseq_lens[train.FORMS], self.word_ids: word_ids[train.FORMS], self.charseq_ids: charseq_ids[train.FORMS]} for tag in args.tags: feeds[self.tags[tag]] = word_ids[train.FACTORS_MAP[tag]] predictions, _ = self.session.run([self.predictions, self.update_loss], feeds) for i in range(len(sentence_lens)): overrides = [None] * dataset.FACTORS for tag in args.tags: overrides[dataset.FACTORS_MAP[tag]] = predictions[tag][i] dataset.write_sentence(conllu, sentences, overrides) sentences += 1 metrics = conll18_ud_eval.evaluate(dataset_conllu, conll18_ud_eval.load_conllu(io.StringIO(conllu.getvalue()))) self.session.run(self.summaries[dataset_name], dict((self.metrics[metric], metrics[metric].f1) for metric in self.METRICS)) return metrics["LAS"].f1 if metrics["LAS"].f1 < 1 else metrics["AllTags"].f1, metrics
def test(self, test_sentences, conllu_ud=None): splits = 64 step = len(test_sentences) / splits start = 0 uas = [] las = [] parsed_sents = [] while start + step < len(test_sentences): u, l, p = self._test_part(test_sentences[start: start + step]) uas.append(u) las.append(l) parsed_sents += p start += step u, l, p = self._test_part(test_sentences[start:]) parsed_sents += p uas.append(u) las.append(l) lenu = len(uas) lenl = len(las) avgu = sum(uas) / lenu avgl = sum(las) / lenl if self.conll_format is const.CONLLU: res = sents_to_conll(parsed_sents) import cStringIO system_ud2 = load_conllu(cStringIO.StringIO(res)) ev = evaluate(conllu_ud, system_ud2) avgu = ev['UAS'].f1 avgl = ev['LAS'].f1 print 'Test results:' print 'uas (no punctuation): ', str(avgu) print 'las (no punctuation): ', str(avgl) return avgu, avgl
def get_UPOS(gold, syst): file1 = open(gold) goldfile = load_conllu(file1) file2 = open(syst) systemfile = load_conllu(file2) return (evaluate(goldfile, systemfile)["UPOS"].f1)
def evaluate(self, dataset_name, dataset, dataset_conllu, args): import io self.session.run(self.reset_metrics) conllu = self.predict(dataset, args) metrics = conll18_ud_eval.evaluate( dataset_conllu, conll18_ud_eval.load_conllu(io.StringIO(conllu))) self.session.run( self.summaries[dataset_name], dict((self.metrics[metric], metrics[metric].f1) for metric in self.METRICS)) return metrics["LAS"].f1 if metrics["LAS"].f1 < 1 else metrics[ "AllTags"].f1, metrics
def evaluate(self, dataset_name, dataset, dataset_conllu, args): import io conllu = self.predict(dataset, True, args) metrics = conll18_ud_eval.evaluate( dataset_conllu, conll18_ud_eval.load_conllu(io.StringIO(conllu))) self.session.run( self.summaries[dataset_name], dict((self.metrics[metric], metrics[metric].f1) for metric in self.METRICS)) if args.parse: return (metrics["LAS"].f1 + metrics["MLAS"].f1 + metrics["BLEX"].f1) / 3., metrics else: return metrics["AllTags"].f1, metrics
def main(): # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("truth", type=str, help="Directory name of the truth dataset.") parser.add_argument("system", type=str, help="Directory name of system output.") parser.add_argument("output", type=str, help="Directory name of the output directory.") args = parser.parse_args() # Load input dataset metadata.json with open(args.truth + "/metadata.json", "r") as metadata_file: metadata = json.load(metadata_file) # Evaluate and compute sum of all treebanks metrics = [ "Tokens", "Sentences", "Words", "UPOS", "XPOS", "UFeats", "AllTags", "Lemmas", "UAS", "LAS", "CLAS", "MLAS", "BLEX" ] treebanks = 0 summation = {} results = [] results_las, results_mlas, results_blex = {}, {}, {} for entry in metadata: treebanks += 1 ltcode, goldfile, outfile = "_".join( (entry['lcode'], entry['tcode'])), entry['goldfile'], entry['outfile'] # Load gold data try: gold = load_conllu_file(args.truth + "/" + goldfile) except: results.append( (ltcode + "-Status", "Error: Cannot load gold file")) continue # Load system data try: system = load_conllu_file(args.system + "/" + outfile) except UDError as e: if e.args[0].startswith("There is a cycle"): results.append( (ltcode + "-Status", "Error: There is a cycle in generated CoNLL-U file")) continue if e.args[0].startswith("There are multiple roots"): results.append(( ltcode + "-Status", "Error: There are multiple roots in a sentence in generated CoNLL-U file" )) continue results.append(( ltcode + "-Status", "Error: There is a format error (tabs, ID values, etc) in generated CoNLL-U file" )) continue except: results.append((ltcode + "-Status", "Error: Cannot open generated CoNLL-U file")) continue # Check for correctness if not system.characters: results.append( (ltcode + "-Status", "Error: The system file is empty")) continue if system.characters != gold.characters: results.append(( ltcode + "-Status", "Error: The concatenation of tokens in gold file and in system file differ, system file has {} nonspace characters, which is approximately {}% of the gold file" .format( len(system.characters), int(100 * len(system.characters) / len(gold.characters))))) continue # Evaluate try: evaluation = evaluate(gold, system) except: # Should not happen results.append(( ltcode + "-Status", "Error: Cannot evaluate generated CoNLL-U file, internal error" )) continue # Generate output metrics and compute sum results.append(( ltcode + "-Status", "OK: Result F1 scores rounded to 5% are LAS={:.0f}% MLAS={:.0f}% BLEX={:.0f}%" .format(100 * round_score(evaluation["LAS"].f1), 100 * round_score(evaluation["MLAS"].f1), 100 * round_score(evaluation["BLEX"].f1)))) for metric in metrics: results.append((ltcode + "-" + metric + "-F1", "{:.9f}".format(100 * evaluation[metric].f1))) summation[metric] = summation.get(metric, 0) + evaluation[metric].f1 results_las[ltcode] = evaluation["LAS"].f1 results_mlas[ltcode] = evaluation["MLAS"].f1 results_blex[ltcode] = evaluation["BLEX"].f1 # Compute averages for metric in reversed(metrics): results.insert(0, ("total-" + metric + "-F1", "{:.9f}".format( 100 * summation.get(metric, 0) / treebanks))) # Generate evaluation.prototext with open(args.output + "/evaluation.prototext", "w") as evaluation: for key, value in results: print('measure{{\n key: "{}"\n value: "{}"\n}}'.format( key, value), file=evaluation) # Generate LAS-F1, MLAS-F1, BLEX-F1 + Status on stdout, Status on stderr for key, value in results: if not key.endswith("-Status"): continue ltcode = key[:-len("-Status")] print("{:13} LAS={:10.6f}% MLAS={:10.6f}% BLEX={:10.6f}% ({})".format( ltcode, 100 * results_las.get(ltcode, 0.), 100 * results_mlas.get(ltcode, 0.), 100 * results_blex.get(ltcode, 0.), value), file=sys.stdout) print("{:13} {}".format(ltcode, value), file=sys.stderr)
for s in nltk.corpus.dependency_treebank.sents()[:200]: sent = detok.detokenize(s) corenlp_model = client.annotate(sent) corenlp_conll_en += corenlp_model # + '\r\n' f_corenlp_en = io.StringIO( corenlp_conll_en.replace("Corp.", "Corp").replace("Conn.", "Conn").replace( "Â", "").replace("Ltd.", "Ltd")) corenlp_en_eval = load_conllu(f_corenlp_en) f_gold_en = io.StringIO( gold_conll_en.replace("Corp.", "Corp").replace("Conn.", "Conn").replace("Ltd.", "Ltd")) gold_en_eval = load_conllu(f_gold_en) corenlp_en_evaluation = evaluate(gold_en_eval, corenlp_en_eval) print_results( corenlp_en_evaluation, "Results for Penn Treebank dataset using CoreNLP Dependency Parser") # Spanish spanish_dep_file = '../../dependency/UD_Spanish-AnCora-master/es_ancora-ud-test.conllu' with open(spanish_dep_file, 'r') as ancora_f: ancora_text = conll_text_reader(ancora_f) corenlp_conll_es = "" with CoreNLPClient( annotators=['tokenize', 'ssplit', 'pos', 'parse', 'depparse'], output_format="conllu",
def conll_eval(system_file, gold_file): gold_ud = load_conllu_file(gold_file) system_ud = load_conllu_file(system_file) return evaluate(gold_ud, system_ud)
sent = detok.detokenize(s) doc = nlp(sent) for s in CoNLL.convert_dict(doc.to_dict()): for w in s: for i, content in enumerate(w): stanza_conll_en += content + '\t' stanza_conll_en = stanza_conll_en[:-1] + '\r\n' stanza_conll_en += '\r\n' f_gold_en = io.StringIO(gold_conll_en) f_stanza_en = io.StringIO(stanza_conll_en) gold_en_eval = load_conllu(f_gold_en) stanza_en_eval = load_conllu(f_stanza_en) stanza_en_evaluation = evaluate(gold_en_eval, stanza_en_eval) print_results(stanza_en_evaluation, "Results for Penn Treebank dataset using Stanza Dependency Parser") # Spanish spanish_dep_file = '../../dependency/UD_Spanish-AnCora-master/es_ancora-ud-test.conllu' with open(spanish_dep_file, 'r') as ancora_f: ancora_text = conll_text_reader(ancora_f) nlp = stanza.Pipeline(processors='tokenize,mwt,pos,lemma,depparse', tokenize_pretokenized=True, lang='es') doc = nlp(ancora_text) stanza_conll_es = "" for s in CoNLL.convert_dict(doc.to_dict()): for w in s: