def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') flags.mark_flag_as_required('prediction_file') predDomain_list, predIntent_list, domain_list, right_intent_num, right_slot_num, exact_num = score_lib.read_data( FLAGS.prediction_file, FLAGS.do_lower_case) logging.info(f'Read file: {FLAGS.prediction_file}') all_num = len(domain_list) domain_acc = score_lib.compute_exact_score(predDomain_list, domain_list) intent_acc = float(right_intent_num) / all_num slot_acc = float(right_slot_num) / all_num exact_score = float(exact_num) / all_num print( 'Num=%d, domain_acc=%.4f, intent_acc=%.4f, slot_acc=%.5f, exact_score=%.4f' % (all_num, domain_acc, intent_acc, slot_acc, exact_score))
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') flags.mark_flag_as_required('prediction_file') sources, predictions, target_lists = score_lib.read_data( FLAGS.prediction_file, FLAGS.case_insensitive) logging.info(f'Read file: {FLAGS.prediction_file}') exact = score_lib.compute_exact_score(predictions, target_lists) sari, keep, addition, deletion = score_lib.compute_sari_scores( sources, predictions, target_lists) print(f'Exact score: {100*exact:.3f}') print(f'SARI score: {100*sari:.3f}') print(f' KEEP score: {100*keep:.3f}') print(f' ADDITION score: {100*addition:.3f}') print(f' DELETION score: {100*deletion:.3f}')
def test_exact_score(self): predictions = ['a b c', 'd ef'] targets = [['x', 'a b c'], ['d e f']] exact = score_lib.compute_exact_score(predictions, targets) # First prediction should match but the second should not. self.assertEqual(exact, 0.5)