def train_test(train_passages, dev_passages, test_passages, args, model_suffix=""): """ Train and test parser on given passage :param train_passages: passage to train on :param dev_passages: passages to evaluate on every iteration :param test_passages: passages to test on after training :param args: extra argument :param model_suffix: string to append to model filename before file extension :return: generator of Scores objects: dev scores for each training iteration (if given dev), and finally test scores """ model_files = [base + model_suffix + ext for base, ext in map(os.path.splitext, args.models or (args.classifier,))] p = Parser(model_files=model_files, config=Config(), beam=args.beam) yield from filter(None, p.train(train_passages, dev=dev_passages, test=test_passages, iterations=args.iterations)) if test_passages: if args.train or args.folds: print("Evaluating on test passages") passage_scores = [] evaluate = args.evaluate or train_passages for result in p.parse(test_passages, evaluate=evaluate, write=args.write): _, *score = result passage_scores += score if passage_scores: scores = Scores(passage_scores) if args.verbose <= 1 or len(passage_scores) > 1: print("\nAverage %s F1 score on test: %.3f" % (get_eval_type(scores), average_f1(scores))) print("Aggregated scores:") scores.print() print_scores(scores, args.testscores) yield scores
def main_generator(): args = Config().args assert args.passages or args.train, "Either passages or --train is required (use -h for help)" assert args.models or args.train or args.folds, "Either --model or --train or --folds is required" assert not (args.train or args.dev) or not args.folds, "--train and --dev are incompatible with --folds" assert args.train or not args.dev, "--dev is only possible together with --train" if args.folds: fold_scores = [] all_passages = list(read_passages(args, args.passages)) assert len(all_passages) >= args.folds, \ "%d folds are not possible with only %d passages" % (args.folds, len(all_passages)) Config().random.shuffle(all_passages) folds = [all_passages[i::args.folds] for i in range(args.folds)] for i in range(args.folds): print("Fold %d of %d:" % (i + 1, args.folds)) dev_passages = folds[i] test_passages = folds[(i + 1) % args.folds] train_passages = [passage for fold in folds if fold is not dev_passages and fold is not test_passages for passage in fold] s = list(train_test(train_passages, dev_passages, test_passages, args, "_%d" % i)) if s and s[-1] is not None: fold_scores.append(s[-1]) if fold_scores: scores = Scores(fold_scores) print("Average test F1 score for each fold: " + ", ".join("%.3f" % average_f1(s) for s in fold_scores)) print("Aggregated scores across folds:\n") scores.print() yield scores else: # Simple train/dev/test by given arguments train_passages, dev_passages, test_passages = [read_passages(args, arg) for arg in (args.train, args.dev, args.passages)] yield from train_test(train_passages, dev_passages, test_passages, args)
def main_generator(): args = Config().args assert args.passages or args.train, "Either passages or --train is required (use -h for help)" assert args.models or args.train or args.folds, "Either --model or --train or --folds is required" assert not (args.train or args.dev) or not args.folds, "--train and --dev are incompatible with --folds" assert args.train or not args.dev, "--dev is only possible together with --train" if args.folds: fold_scores = [] all_passages = list(read_passages(args, args.passages)) assert len(all_passages) >= args.folds, \ "%d folds are not possible with only %d passages" % (args.folds, len(all_passages)) Config().random.shuffle(all_passages) folds = [all_passages[i::args.folds] for i in range(args.folds)] for i in range(args.folds): print("Fold %d of %d:" % (i + 1, args.folds)) dev_passages = folds[i] test_passages = folds[(i + 1) % args.folds] train_passages = [passage for fold in folds if fold is not dev_passages and fold is not test_passages for passage in fold] s = list(train_test(train_passages, dev_passages, test_passages, args, "_%d" % i)) if s and s[-1] is not None: fold_scores.append(s[-1]) if fold_scores: scores = Scores(fold_scores) print("Average test F1 score for each fold: " + ", ".join("%.3f" % average_f1(s) for s in fold_scores)) print("Aggregated scores across folds:\n") scores.print() yield scores else: # Simple train/dev/test by given arguments train_passages, dev_passages, test_passages = [read_passages(args, arg) for arg in (args.train, args.dev, args.passages)] yield from train_test(train_passages, dev_passages, test_passages, args)
def train_test(train_passages, dev_passages, test_passages, args, model_suffix=""): """ Train and test parser on given passage :param train_passages: passage to train on :param dev_passages: passages to evaluate on every iteration :param test_passages: passages to test on after training :param args: extra argument :param model_suffix: string to append to model filename before file extension :return: generator of Scores objects: dev scores for each training iteration (if given dev), and finally test scores """ model_files = [base + model_suffix + ext for base, ext in map(os.path.splitext, args.models or (args.classifier,))] p = Parser(model_files=model_files, config=Config(), beam=args.beam) yield from filter(None, p.train(train_passages, dev=dev_passages, test=test_passages, iterations=args.iterations)) if test_passages: if args.train or args.folds: print("Evaluating on test passages") passage_scores = [] evaluate = args.evaluate or train_passages for result in p.parse(test_passages, evaluate=evaluate, write=args.write): _, *score = result passage_scores += score if passage_scores: scores = Scores(passage_scores) if args.verbose <= 1 or len(passage_scores) > 1: print("\nAverage %s F1 score on test: %.3f" % (get_eval_type(scores), average_f1(scores))) print("Aggregated scores:") scores.print() print_scores(scores, args.testscores) yield scores
def eval(self, passages, mode, scores_filename, display=True): print("Evaluating on %s passages" % mode.name) passage_scores = [s for _, s in self.parse(passages, mode=mode, evaluate=True, display=display)] scores = Scores(passage_scores) average_score = average_f1(scores) prefix = ".".join(map(str, [self.iteration, self.epoch] + ( [self.batch] if self.config.args.save_every else []))) if display: print("Evaluation %s, average %s F1 score on %s: %.3f%s" % (prefix, get_eval_type(scores), mode.name, average_score, scores.details(average_f1))) print_scores(scores, scores_filename, prefix=prefix, prefix_title="iteration") return average_score, scores
def main(args): for spec in read_specs(args, converters=FROM_FORMAT): scores = [] if not args.verbose: spec.passages = tqdm( spec.passages, unit=" passages", desc="Parsing " + (spec.out_dir if spec.out_dir != "." else spec.lang)) for passage, parsed in parse(spec.passages, spec.lang, spec.udpipe, args.verbose): map_labels(parsed, args.label_map) normalize(parsed, extra=True) if args.write: write_passage(parsed, args) if args.evaluate: evaluator = EVALUATORS.get(args.output_format) converter = TO_FORMAT.get(args.output_format) if converter is not None: passage, parsed = map(converter, (passage, parsed)) if evaluator is not None: scores.append( evaluator.evaluate(parsed, passage, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): for spec in read_specs(args, converters=CONVERTERS): scores = [] sentences, to_parse = tee( (to_conllu_native(p), to_conllu_native(p, test=True, enhanced=False) ) if isinstance(p, core.Passage) else (p, strip_enhanced(p)) for p in spec.passages) t = tqdm(zip((x for x, _ in sentences), split_by_empty_lines( udpipe((x for _, x in to_parse), spec.udpipe, args.verbose))), unit=" sentences") for sentence, parsed in t: sentence = list(sentence) if args.write: i = next(find_ids(sentence)) t.set_postfix(id=i) with open_out_file(spec, i) as f: for line in parsed: print(line, file=f) if args.evaluate: scores.append( evaluate(parsed, sentence, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def diff(results, indices=None, verbose=False): scores = [Scores(r if indices is None else [r[i] for i in indices]) for r in results] fields = np.array([s.fields() for s in scores], dtype=float) if verbose: print(" ".join(evaluation.Scores.field_titles())) print("\n".join(map(str, fields))) return fields[1] - fields[0]
def test_parser(config, model_type, formats, default_setting, text=True): filename = "test_files/models/%s_%s%s" % ("_".join(formats), model_type, default_setting.suffix()) remove_existing(filename) config.update(default_setting.dict()) scores = [] params = [] passages = list(map(load_passage, passage_files(*formats))) evaluate = ("amr" not in formats) for mode in "train", "load": print("-- %sing %s" % (mode, model_type)) config.update(dict(classifier=model_type, copy_shared=None)) p = Parser(model_files=filename, config=config) p.save_init = True list( p.train(passages if mode == "train" else None, dev=passages, test=True, iterations=2)) assert p.model.is_finalized, "Model should be finalized after %sing" % mode assert not getattr(p.model.feature_extractor, "node_dropout", 0), p.model.feature_extractor.node_dropout all_params = p.model.all_params() params.append(all_params) param1, param2 = [ d.get("W") for d in (all_params, p.model.feature_extractor.params) ] if param1 is not None and param2 and param2.init is not None and not config.args.update_word_vectors: assert_allclose(param1, weight_decay(p.model) * param2.init, rtol=1e-6) text_results = results = list(p.parse(passages, evaluate=evaluate)) if text: print("Converting to text and parsing...") text_results = list( p.parse([ p3 for p1 in passages for p2 in convert.to_text(p1, sentences=False) for p3 in convert.from_text( p2, p1.ID, extra_format=p1.extra.get("format")) ])) assert len(results) == len(text_results) if evaluate: scores.append(Scores(tuple(zip(*results))[1]).average_f1()) if text: for t, (r, s) in zip(text_results, results): print(" %s F1=%.3f" % (r.ID, s.average_f1())) assert not list(p.parse(())) # parsing nothing returns nothing print() assert_all_params_equal(*params) if evaluate: print("-- average f1: %.3f, %.3f\n" % tuple(scores)) assert scores[0] == pytest.approx(scores[1], 0.1)
def eval(self, passages, mode, scores_filename, display=True): print("Evaluating on %s passages" % mode.name) passage_scores = [s for _, s in self.parse(passages, mode=mode, evaluate=True, display=display)] scores = Scores(passage_scores) average_score = average_f1(scores) prefix = ".".join(map(str, [self.iteration, self.epoch] + ( [self.batch] if self.config.args.save_every else []))) if display: print("Evaluation %s, average %s F1 score on %s: %.3f%s" % (prefix, get_eval_type(scores), mode.name, average_score, scores.details(average_f1))) print_scores(scores, scores_filename, prefix=prefix, prefix_title="iteration") return average_score, scores
def main(): argparser = configargparse.ArgParser(description=desc) argparser.add_argument("filenames", nargs="+", help="file names to convert and evaluate") add_verbose_arg(argparser, help="detailed evaluation output") add_boolean_option(argparser, "wikification", "Spotlight to wikify any named node (for AMR)") argparser.add_argument("-o", "--out-dir", help="output directory (if unspecified, files are not written)") args = argparser.parse_args() scores = [] for pattern in args.filenames: filenames = glob(pattern) if not filenames: raise IOError("Not found: " + pattern) for filename in filenames: print("\rConverting '%s'" % filename, end="") if args.out_dir or args.verbose: print(flush=True) basename, ext = os.path.splitext(os.path.basename(filename)) passage_format = ext.lstrip(".") converters = CONVERTERS.get(passage_format, CONVERTERS["amr"]) evaluator = EVALUATORS.get(passage_format, EVALUATORS["amr"]).evaluate with open(filename, encoding="utf-8") as f: for passage, ref, passage_id in converters[0](f, passage_id=basename, return_original=True): if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) outfile = "%s/%s.xml" % (args.out_dir, passage.ID) print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) ioutil.passage2file(passage, outfile) try: guessed = converters[1](passage, wikification=args.wikification, use_original=False) except Exception as e: raise ValueError("Error converting %s back from %s" % (filename, passage_format)) from e if args.out_dir: outfile = "%s/%s%s" % (args.out_dir, passage.ID, ext) print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) with open(outfile, "w", encoding="utf-8") as f_out: print("\n".join(guessed), file=f_out) try: s = evaluator(guessed, ref, verbose=args.verbose > 1) except Exception as e: raise ValueError("Error evaluating conversion of %s" % filename) from e scores.append(s) if args.verbose: print(passage_id) s.print() print() if args.verbose and len(scores) > 1: print("Aggregated scores:") Scores(scores).print() sys.exit(0)
def main(args): for passages, out_dir, lang in read_specs(args): scores = [] if not args.verbose: passages = tqdm(passages, unit=" passages", desc="Parsing " + out_dir) for passage, parsed in ANNOTATORS[args.parser](passages, lang, args.verbose): if args.write: write_passage(parsed, args) else: map_labels(parsed, args.label_map) if args.evaluate: evaluator = EVALUATORS[args.output_format] _, converter = CONVERTERS[args.output_format] if converter is not None: passage, parsed = map(converter, (passage, parsed)) scores.append(evaluator.evaluate(parsed, passage, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): scores = [] for pattern in args.filenames: filenames = glob(pattern) if not filenames: raise IOError("Not found: " + pattern) for filename in filenames: print("\rConverting '%s'" % filename, end="") if args.out_dir or args.verbose: print(flush=True) basename, ext = os.path.splitext(os.path.basename(filename)) passage_format = ext.lstrip(".") converters = CONVERTERS.get(passage_format, CONVERTERS["amr"]) evaluator = EVALUATORS.get(passage_format, EVALUATORS["amr"]).evaluate with open(filename, encoding="utf-8") as f: for passage, ref, passage_id in converters[0](f, passage_id=basename, return_original=True): if args.normalize: normalize(passage, extra=args.extra_normalization) if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) outfile = "%s/%s.xml" % (args.out_dir, passage.ID) print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) ioutil.passage2file(passage, outfile) try: guessed = converters[1](passage, wikification=args.wikification, use_original=False) except Exception as e: raise ValueError("Error converting %s back from %s" % (filename, passage_format)) from e if args.out_dir: outfile = "%s/%s%s" % (args.out_dir, passage.ID, ext) print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) with open(outfile, "w", encoding="utf-8") as f_out: print("\n".join(guessed), file=f_out) try: s = evaluator(guessed, ref, verbose=args.verbose > 1) except Exception as e: raise ValueError("Error evaluating conversion of %s" % filename) from e scores.append(s) if args.verbose: print(passage_id) s.print() print() if args.verbose and len(scores) > 1: print("Aggregated scores:") Scores(scores).print()
def main(args): for spec in read_specs(args, converters=CONVERTERS): scores = [] sentences1, sentences2 = tee(spec.passages) t = tqdm(zip( sentences1, split_by_empty_lines(udpipe(sentences2, spec.udpipe, args.verbose))), unit=" sentences") for sentence, parsed in t: sentence = list(sentence) if args.write: i = next(find_ids(sentence)) t.set_postfix(id=i) with open(os.path.join(spec.out_dir, i + ".conllu"), "w", encoding="utf-8") as f: for line in parsed: print(line, file=f) if args.evaluate: scores.append( evaluate(parsed, sentence, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) scores = [] for pattern in args.filenames: for filename in glob(pattern) or [pattern]: file_scores = [] basename, ext = os.path.splitext(os.path.basename(filename)) passage_format = ext.lstrip(".") if passage_format == "txt": passage_format = args.format in_converter, out_converter = CONVERTERS.get( passage_format, CONVERTERS[args.format]) evaluate = EVALUATORS.get(passage_format, EVALUATORS[args.format]) with open(filename, encoding="utf-8") as f: t = tqdm(in_converter(f, passage_id=basename, return_original=True), unit=" passages", desc=("Converting '%s'" % filename) + ((", writing to '%s'" % args.out_dir) if args.out_dir else "")) for passage, ref, passage_id in t: if args.normalize: normalize(passage, extra=args.extra_normalization) if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) outfile = os.path.join(args.out_dir, passage.ID + ".xml") if args.verbose: with ioutil.external_write_mode(): print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) ioutil.passage2file(passage, outfile) try: guessed = out_converter(passage, wikification=args.wikification, use_original=False) except Exception as e: raise ValueError("Error converting %s back from %s" % (filename, passage_format)) from e if args.out_dir: outfile = os.path.join(args.out_dir, passage.ID + ext) if args.verbose: with ioutil.external_write_mode(): print("Writing '%s'..." % outfile, file=sys.stderr, flush=True) with open(outfile, "w", encoding="utf-8") as f_out: print("\n".join(guessed), file=f_out) try: s = evaluate(guessed, ref, verbose=args.verbose > 1, units=args.units) except Exception as e: raise ValueError("Error evaluating conversion of %s" % filename) from e file_scores.append(s) if args.verbose: with ioutil.external_write_mode(): print(passage_id) s.print() t.set_postfix(F1="%.2f" % (100.0 * Scores(file_scores).average_f1())) scores += file_scores print() if args.verbose and len(scores) > 1: print("Aggregated scores:") Scores(scores).print()
if not os.path.isdir(os.path.join(d, f)) ] if os.path.isdir(d) else [d] for d in (os.path.join(submission_dir, track, lang), os.path.join(truth_dir, lang), None) ] evaluate = EVALUATORS.get( passage_format(files[1][0])[1], EVALUATORS["amr"]) results = list( evaluate_all(evaluate, files, format="amr", name="Evaluating", unlabeled=False, matching_ids=True)) summary = Scores(results) # write results to html file and append to values output_html_file.write("<tr>\n" "<td>%s</td>" % competition) # labeled output_html_file.write("<td>%.3f</td>\n" % (summary.average_f1(LABELED))) values.append(round(summary.average_f1(LABELED), 3)) for (title, field) in zip(summary.titles(LABELED), summary.fields(LABELED)): output_html_file.write("<td>%.3f</td>\n" % (float(field))) values.append(float(field))