def mk_test_report(hconf, dconf):
"""Generate reports for test data"""
econf = hconf.test_evaluation
if econf is None:
return
p_path = hconf.decode_output_path(econf, None)
slices = [
Slice(fold=None,
configuration=_report_key(econf),
predictions=load_predictions(p_path),
enable_details=True)
]
_mk_report(hconf, dconf, slices, None, test_data=True)
_copy_version_files(hconf, True)
report_dir = hconf.report_dir_path(True, fold=None)
final_report_dir = hconf.report_dir_path(True, fold=None, is_tmp=False)
mk_test_graphs(hconf, dconf)
_mk_hashfile(hconf, dconf, True)
# this can happen if resuming a report; better copy
# it again
if fp.exists(final_report_dir):
shutil.rmtree(final_report_dir)
shutil.copytree(report_dir, final_report_dir)
print('TEST Report saved in ', final_report_dir, file=sys.stderr)
def _mk_econf_graphs(hconf, edus, gold, econf, fold):
"Return jobs generating graphs for a single configuration"
predictions = load_predictions(hconf.decode_output_path(econf, fold))
for diffmode in GraphDiffMode:
# output path
if diffmode == GraphDiffMode.solo:
output_bn_prefix = 'graphs-'
elif diffmode == GraphDiffMode.diff:
output_bn_prefix = 'graphs-gold-vs-'
elif diffmode == GraphDiffMode.diff_intra:
output_bn_prefix = 'graphs-sent-gold-vs-'
else:
raise Exception('Unknown diff mode {}'.format(diffmode))
want_test = fold is None
suffix = 'test' if want_test\
else fp.basename(hconf.fold_dir_path(fold))
output_dir = fp.join(hconf.report_dir_path(want_test, None),
output_bn_prefix + suffix, econf.key)
# settings
to_hide = 'inter' if diffmode == GraphDiffMode.diff_intra else None
settings =\
GraphSettings(hide=to_hide,
select=hconf.graph_docs,
unrelated=False,
timeout=15,
quiet=False)
if diffmode == GraphDiffMode.solo:
yield delayed(graph_all)(edus, predictions, settings, output_dir)
else:
yield delayed(diff_all)(edus, gold, predictions, settings,
output_dir)
def mk_test_report(hconf, dconf):
"Generate reports for test data"
econf = hconf.test_evaluation
if econf is None:
return
p_path = hconf.decode_output_path(econf, None)
slices = [Slice(fold=None,
configuration=_report_key(econf),
predictions=load_predictions(p_path),
enable_details=True)]
_mk_report(hconf, dconf, slices, None,
test_data=True)
_copy_version_files(hconf, True)
report_dir = hconf.report_dir_path(True, fold=None)
final_report_dir = hconf.report_dir_path(True, fold=None, is_tmp=False)
mk_test_graphs(hconf, dconf)
_mk_hashfile(hconf, dconf, True)
# this can happen if resuming a report; better copy
# it again
if fp.exists(final_report_dir):
shutil.rmtree(final_report_dir)
shutil.copytree(report_dir, final_report_dir)
print('TEST Report saved in ', final_report_dir,
file=sys.stderr)
def _fold_report_slices(hconf, fold):
"""
Report slices for a given fold
"""
print('Scoring fold {}...'.format(fold), file=sys.stderr)
dkeys = [econf.key for econf in hconf.detailed_evaluations]
for econf in hconf.evaluations:
p_path = hconf.decode_output_path(econf, fold)
yield Slice(fold=fold,
configuration=_report_key(econf),
predictions=load_predictions(p_path),
enable_details=econf.key in dkeys)
def _fold_report_slices(hconf, fold):
"""
Report slices for a given fold
"""
print('Scoring fold {}...'.format(fold),
file=sys.stderr)
dkeys = [econf.key for econf in hconf.detailed_evaluations]
for econf in hconf.evaluations:
p_path = hconf.decode_output_path(econf, fold)
yield Slice(fold=fold,
configuration=_report_key(econf),
predictions=load_predictions(p_path),
enable_details=econf.key in dkeys)
def _fold_report_slices(lconf, fold):
"""
Report slices for a given fold
"""
print('Scoring fold {}...'.format(fold),
file=sys.stderr)
dkeys = [econf.key for econf in DETAILED_EVALUATIONS]
for econf in EVALUATIONS:
p_path = decode_output_path(lconf, econf, fold)
enable_details = econf.key in dkeys
stripped_decoder_key = econf.decoder.key[len(econf.settings.key) + 1:]
config = (econf.learner.key,
stripped_decoder_key,
econf.settings.key)
yield Slice(fold, config,
load_predictions(p_path),
enable_details)
def _mk_econf_graphs(hconf, edus, gold, econf, fold):
"Return jobs generating graphs for a single configuration"
predictions = load_predictions(hconf.decode_output_path(econf, fold))
for diffmode in GraphDiffMode:
# output path
if diffmode == GraphDiffMode.solo:
output_bn_prefix = 'graphs-'
elif diffmode == GraphDiffMode.diff:
output_bn_prefix = 'graphs-gold-vs-'
elif diffmode == GraphDiffMode.diff_intra:
output_bn_prefix = 'graphs-sent-gold-vs-'
else:
raise Exception('Unknown diff mode {}'.format(diffmode))
want_test = fold is None
suffix = 'test' if want_test\
else fp.basename(hconf.fold_dir_path(fold))
output_dir = fp.join(hconf.report_dir_path(want_test, None),
output_bn_prefix + suffix,
econf.key)
# settings
to_hide = 'inter' if diffmode == GraphDiffMode.diff_intra else None
settings =\
GraphSettings(hide=to_hide,
select=hconf.graph_docs,
unrelated=False,
timeout=15,
quiet=False)
if diffmode == GraphDiffMode.solo:
yield delayed(graph_all)(edus,
predictions,
settings,
output_dir)
else:
yield delayed(diff_all)(edus,
gold,
predictions,
settings,
output_dir)