def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
assert FLAGS.dataset
assert len(FLAGS.reference_files) == len(FLAGS.prediction_files)
for reference_file, prediction_file in zip(FLAGS.reference_files,
FLAGS.prediction_files):
with tf.io.gfile.GFile(prediction_file, 'r') as input_file:
predictions = {}
for line in input_file:
line = line.strip()
if line:
segments = line.split('\t')
key = segments[0]
value = segments[1:]
predictions[key] = text_utils.normalize_answers(value)
with tf.io.gfile.GFile(reference_file, 'r') as input_file:
reference = json.load(input_file)
references = {}
for index, data in enumerate(reference):
key = '%s-%d' % (FLAGS.dataset, index)
references[key] = text_utils.normalize_answers(data)
num_correct = 0
for key, gold_answer in references.items():
pred_answer = predictions[key]
is_correct = gold_answer == pred_answer
if is_correct:
num_correct += 1
print('Correct: ', num_correct, len(references),
num_correct / float(len(references)))
def _get_pred_denotation_result(example):
"""Computes predicted denotation."""
if example.pred_agg_function is None:
raise ValueError('pred_agg_function is None')
if example.pred_cell_coo is None:
raise ValueError('pred_cell_coo is None')
agg_function = example.pred_agg_function
cell_coo = example.pred_cell_coo
denotation, values = execute(agg_function, cell_coo, example.table)
denotation = _to_float32s(denotation)
denotation = text_utils.normalize_answers(denotation)
return DenotationResult(
denotation=denotation,
values=values,
agg_function=agg_function,
cell_coordinates=cell_coo,
)
def _get_gold_denotation_result(example):
"""Computes gold denotation of the example."""
if not example.has_gold_answer:
# No gold answer for this example.
return None
agg_function = example.gold_agg_function
cell_coo = example.gold_cell_coo
if example.float_answer is None:
denotation, values = execute(agg_function, cell_coo, example.table)
elif math.isnan(example.float_answer):
denotation = []
values = []
else:
denotation = [(example.float_answer)]
values = []
denotation = _to_float32s(denotation)
denotation = text_utils.normalize_answers(denotation)
return DenotationResult(
denotation=denotation,
values=values,
agg_function=agg_function,
cell_coordinates=cell_coo,
)
def test_normalize_answers(self, value, expected):
self.assertEqual(expected, text_utils.normalize_answers(value))