def test_calc_denotation_accuracy_without_gold_answer(self):
table = pd.DataFrame([['a', 'b'], ['0', '1']], columns=['A', 'B'])
denotation_accuracy = calc_metrics_utils.calc_denotation_accuracy(
examples={
'0':
calc_metrics_utils.Example(
example_id='0',
question='q',
table_id='tab_0',
table=table,
gold_cell_coo=set(),
gold_agg_function=calc_metrics_utils._AggregationFunction.
NONE,
float_answer=None,
has_gold_answer=False,
),
'1':
calc_metrics_utils.Example(
example_id='1',
question='q',
table_id='tab_0',
table=table,
gold_cell_coo={(0, 0)},
gold_agg_function=calc_metrics_utils._AggregationFunction.
NONE,
float_answer=None,
has_gold_answer=True,
pred_cell_coo={(0, 0)},
)
},
denotation_errors_path=None,
predictions_file_name=None)
self.assertEqual(0.5, denotation_accuracy)
def _read_data_examples(data_path):
"""Reads examples from a dataset csv file."""
data_examples = {}
with tf.io.gfile.GFile(data_path, mode='r') as f:
reader = csv.DictReader(f, delimiter='\t')
for row in reader:
ex_id = '{}-{}_{}'.format(row['id'], row['annotator'],
row['position'])
question = row['question'].strip()
table_id = row['table_file']
gold_cell_coo = prediction_utils.parse_coordinates(
row['answer_coordinates'])
gold_agg_function = int(row['aggregation'])
float_answer_raw = row['float_answer']
float_answer = float(
float_answer_raw) if float_answer_raw else None
ex = calc_metrics_utils.Example(ex_id,
question,
table_id,
None,
gold_cell_coo,
gold_agg_function,
float_answer,
has_gold_answer=True)
data_examples[ex_id] = ex
return data_examples
def test_read_predictions_without_pred_aggr(self):
predictions_path = _write_predictions(
data=[['dev-0', '0', '0', '["(0,0)"]']],
headers=('id', 'annotator', 'position', 'answer_coordinates'))
examples = {
'dev-0-0_0':
calc_metrics_utils.Example(
example_id='dev-0-0_0',
question='q',
table_id='tab_0',
table=pd.DataFrame(),
gold_cell_coo={},
gold_agg_function=calc_metrics_utils._AggregationFunction.NONE,
float_answer=None,
has_gold_answer=True,
)
}
calc_metrics_utils.read_predictions(predictions_path, examples)
self.assertLen(examples, 1)
self.assertEqual(
next(iter(examples.values())).pred_agg_function,
calc_metrics_utils._AggregationFunction.NONE)