def eval_taggers_igt(ef: ExperimentFiles, overwrite = False):
evaluation_performed = False
for tc in ef.tagger_configs():
if not os.path.exists(tc.rgigt_eval()) or overwrite:
if not evaluation_performed:
REPRO_LOG.log(NORM_LEVEL, "Evaluating POS taggers on IGT data.")
if USE_CONDOR:
condorify(*tc.eval_igt_pos_args(True))
else:
args, kwargs = tc.eval_igt_pos_args(False)
evaluate_intent(args, **kwargs)
evaluation_performed = True
if USE_CONDOR and evaluation_performed:
if condor_email:
condor.condor_wait_notify("IGT Evaluation Complete", condor_email, subject="IGT POS Evaluation Complete")
else:
condor.condor_wait()
# -------------------------------------------
# Now, consolidate all of the different methods and report.
# -------------------------------------------
te = TaggerEval()
for tc in ef.tagger_configs():
if not os.path.exists(tc.rgigt_eval()):
REPRO_LOG.warn("Missing eval: {}".format(os.path.basename(tc.rgigt_eval())))
else:
with open(tc.rgigt_eval(), 'r') as f:
data = f.readlines()
filename, matches, compares, acc = data[-1].strip().split(',')
te.add_tc(tc, int(matches), int(compares))
print(te)
nfold=args.nfold)
# FILTER
elif args.subcommand == CMD_FILTER:
filter_corpus(flatten_list(getattr(args, ARG_INFILE)), getattr(args, ARG_OUTFILE), **vars(args))
# EXTRACT
elif args.subcommand == CMD_EXTRACT:
extract_from_xigt(input_filelist=flatten_list(args.FILE), **vars(args))
# EVAL
elif args.subcommand == CMD_EVAL:
evaluate_intent(flatten_list(args.FILE),
eval_alignment=args.alignment,
eval_ds=args.ds_projection,
eval_posproj=args.pos_projection,
classifier_path=args.classifier,
classifier_feats=args.classifier_feats,
eval_tagger=args.pos_tagger,
gold_tagmap=args.tagmap_gold, trans_tagmap=args.tagmap_trans, outpath=args.output)
# TEXT CONVERT
elif args.subcommand == CMD_TEXT:
xc = text_to_xigtxml(args.FILE)
dump(args.OUT_FILE, xc)
# PROJECT
elif args.subcommand == CMD_PROJECT:
do_projection(**vars(args))
# REPRO
elif args.subcommand == CMD_REPRO:
