def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
flags.mark_flag_as_required('prediction_file')
sources, predictions, _ = score_lib.read_data(FLAGS.prediction_file,
FLAGS.case_insensitive)
ref_filepaths = [
get_data_filepath('turkcorpus', 'valid', 'simple.turk', i)
for i in range(8)
]
target_lists = [read_lines(ref_filepath) for ref_filepath in ref_filepaths]
logging.info(f'Read file: {FLAGS.prediction_file}')
turk_scores = get_all_scores(orig_sents=sources,
sys_sents=predictions,
refs_sents=target_lists)
logging.info("[turk] {}".format(turk_scores))
ref_filepaths = [
get_data_filepath('asset', 'valid', 'simp', i) for i in range(10)
]
target_lists = [read_lines(ref_filepath) for ref_filepath in ref_filepaths]
asset_scores = get_all_scores(orig_sents=sources,
sys_sents=predictions,
refs_sents=target_lists)
logging.info("[asset] {}".format(asset_scores))
def test_input_reading(self):
path = os.path.join(FLAGS.test_tmpdir, 'file.txt')
with tf.io.gfile.GFile(path, 'w') as writer:
writer.write('source\tpred\ttarget1\ttarget2\n'
'Source2\tpRed2\ttarGet3\n')
sources, predictions, target_lists = score_lib.read_data(
path, lowercase=True)
self.assertEqual(sources, ['source', 'source2'])
self.assertEqual(predictions, ['pred', 'pred2'])
self.assertEqual(target_lists, [['target1', 'target2'], ['target3']])
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
flags.mark_flag_as_required('prediction_file')
sources, predictions, target_lists = score_lib.read_data(
FLAGS.prediction_file, FLAGS.case_insensitive)
logging.info(f'Read file: {FLAGS.prediction_file}')
exact = score_lib.compute_exact_score(predictions, target_lists)
sari, keep, addition, deletion = score_lib.compute_sari_scores(
sources, predictions, target_lists)
print(f'Exact score: {100*exact:.3f}')
print(f'SARI score: {100*sari:.3f}')
print(f' KEEP score: {100*keep:.3f}')
print(f' ADDITION score: {100*addition:.3f}')
print(f' DELETION score: {100*deletion:.3f}')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
flags.mark_flag_as_required('prediction_file')
predDomain_list, predIntent_list, domain_list, right_intent_num, right_slot_num, exact_num = score_lib.read_data(
FLAGS.prediction_file, FLAGS.do_lower_case)
logging.info(f'Read file: {FLAGS.prediction_file}')
all_num = len(domain_list)
domain_acc = score_lib.compute_exact_score(predDomain_list, domain_list)
intent_acc = float(right_intent_num) / all_num
slot_acc = float(right_slot_num) / all_num
exact_score = float(exact_num) / all_num
print(
'Num=%d, domain_acc=%.4f, intent_acc=%.4f, slot_acc=%.5f, exact_score=%.4f'
% (all_num, domain_acc, intent_acc, slot_acc, exact_score))
