def _convert_single_wtq(interaction_file, prediction_file, output_file):
"""Convert predictions to WikiTablequestions format."""
interactions = dict(
(prediction_utils.parse_interaction_id(i.id), i)
for i in prediction_utils.iterate_interactions(interaction_file))
missing_interaction_ids = set(interactions.keys())
with tf.io.gfile.GFile(output_file, 'w') as output_file:
for prediction in prediction_utils.iterate_predictions(
prediction_file):
interaction_id = prediction['id']
if interaction_id in missing_interaction_ids:
missing_interaction_ids.remove(interaction_id)
else:
continue
coordinates = prediction_utils.parse_coordinates(
prediction['answer_coordinates'])
denot_pred, _ = calc_metrics_utils.execute(
int(prediction.get('pred_aggr', '0')), coordinates,
prediction_utils.table_to_panda_frame(
interactions[interaction_id].table))
answers = '\t'.join(sorted(map(str, denot_pred)))
output_file.write('{}\t{}\n'.format(interaction_id, answers))
for interaction_id in missing_interaction_ids:
output_file.write('{}\n'.format(interaction_id))
def _read_data_examples(data_path):
"""Reads examples from a dataset csv file."""
data_examples = {}
with tf.io.gfile.GFile(data_path, mode='r') as f:
reader = csv.DictReader(f, delimiter='\t')
for row in reader:
ex_id = '{}-{}_{}'.format(row['id'], row['annotator'],
row['position'])
question = row['question'].strip()
table_id = row['table_file']
gold_cell_coo = prediction_utils.parse_coordinates(
row['answer_coordinates'])
gold_agg_function = int(row['aggregation'])
float_answer_raw = row['float_answer']
float_answer = float(
float_answer_raw) if float_answer_raw else None
ex = calc_metrics_utils.Example(ex_id,
question,
table_id,
None,
gold_cell_coo,
gold_agg_function,
float_answer,
has_gold_answer=True)
data_examples[ex_id] = ex
return data_examples
def read_predictions(predictions_path, examples):
"""Reads predictions from a csv file."""
for row in prediction_utils.iterate_predictions(predictions_path):
pred_id = '{}-{}_{}'.format(row['id'], row['annotator'], row['position'])
example = examples[pred_id]
example.pred_cell_coo = prediction_utils.parse_coordinates(
row['answer_coordinates'])
example.pred_agg_function = int(row.get('pred_aggr', '0'))
def get_predictions(prediction_file):
"""Yields an iterable of Prediction objects from a tsv prediction file."""
fn_map = {
'logits_cls': float,
'position': int,
'answer_coordinates':
lambda x: list(prediction_utils.parse_coordinates(x)),
'answers': token_answers_from_text,
'token_probabilities': json.loads,
}
for prediction_dict in prediction_utils.iterate_predictions(
prediction_file):
for key in tuple(prediction_dict.keys()):
fn = fn_map.get(key, lambda x: x)
prediction_dict[key] = fn(prediction_dict[key])
yield Prediction(**prediction_dict)
def read_predictions(predictions_path, examples):
"""Reads predictions from a csv file."""
for row in prediction_utils.iterate_predictions(predictions_path):
pred_id = '{}-{}_{}'.format(row['id'], row['annotator'],
row['position'])
example = examples[pred_id]
example.pred_cell_coo = prediction_utils.parse_coordinates(
row['answer_coordinates'])
example.pred_agg_function = int(row.get('pred_aggr', '0'))
if 'column_scores' in row:
column_scores = list(
filter(None, row['column_scores'][1:-1].split(' ')))
removed_column_scores = [
float(score) for score in column_scores if float(score) < 0.0
]
if column_scores:
example.weight = len(removed_column_scores) / len(
column_scores)
def predict(table_data, queries):
print("Prediction started!")
table = [
list(map(lambda s: s.strip(), row.split("|")))
for row in table_data.split("\n") if row.strip()
]
examples = convert_interactions_to_examples([(table, queries)])
write_tf_example("results/sqa/tf_examples/test.tfrecord", examples)
write_tf_example("results/sqa/tf_examples/random-split-1-dev.tfrecord", [])
print("Processed table data!")
os.system(''' python tapas/tapas/run_task_main.py \
--task="SQA" \
--output_dir="results" \
--noloop_predict \
--test_batch_size=3 \
--tapas_verbosity="ERROR" \
--compression_type= \
--init_checkpoint="tapas_sqa_base/model.ckpt" \
--bert_config_file="tapas_sqa_base/bert_config.json" \
--mode="predict" 2> error''')
print("Prediction completed!")
results_path = "results/sqa/model/test_sequence.tsv"
all_coordinates = []
answers_lst = []
df = pd.DataFrame(table[1:], columns=table[0])
#display(IPython.display.HTML(df.to_html(index=False)))
print("Result printing!")
with open(results_path) as csvfile:
reader = csv.DictReader(csvfile, delimiter='\t')
for row in reader:
coordinates = prediction_utils.parse_coordinates(
row["answer_coordinates"])
all_coordinates.append(coordinates)
answers = ', '.join(
[table[row + 1][col] for row, col in coordinates])
position = int(row['position'])
print(">", queries[position])
print(answers)
answers_lst.append(answers)
return answers_lst
def predict(table_data, queries):
table = table_data.values.tolist()
examples = convert_interactions_to_examples([(table, queries)])
write_tf_example("results/sqa/tf_examples/test.tfrecord", examples)
write_tf_example("results/sqa/tf_examples/random-split-1-dev.tfrecord", [])
cmd = '/mnt/d/Data_Science_Work/tapas/predict.sh'
subprocess.call(cmd)
results_path = "results/sqa/model/test_sequence.tsv"
all_coordinates = []
with open(results_path) as csvfile:
reader = csv.DictReader(csvfile, delimiter='\t')
for row in reader:
coordinates = prediction_utils.parse_coordinates(row["answer_coordinates"])
all_coordinates.append(coordinates)
answers = ', '.join([table[row + 1][col] for row, col in coordinates])
position = int(row['position'])
print(">", queries[position])
print(answers)
return answers
def eval_cell_selection(
questions,
predictions_file,
):
"""Evaluates cell selection results in HybridQA experiment.
Args:
questions: A map of Question protos by their respective ids.
predictions_file: Path to a tsv file with predictions for a checkpoint.
Yields:
An AnswerType and its corresponding CellSelectionMetrics instance
"""
total = collections.Counter()
total_correct = collections.Counter()
total_correct_at_k = {k: collections.Counter() for k in _RECALL_KS}
total_seen = collections.Counter()
total_non_empty = collections.Counter()
total_coordinates = collections.Counter()
sum_precision = collections.defaultdict(float)
for question in questions.values():
for answer_type in [AnswerType.ALL, _get_answer_type(question)]:
total[answer_type] += 1
for row in prediction_utils.iterate_predictions(predictions_file):
question = questions.get(row['question_id'])
if question is None:
# The dataset lost some examples after an update.
continue
gold_coordinates = {(x.row_index, x.column_index)
for x in question.answer.answer_coordinates}
coordinates = prediction_utils.parse_coordinates(
row['answer_coordinates'])
# We only care about finding one correct cell for the downstream model.
correct_coordinates = len(coordinates & gold_coordinates)
has_probabilities = 'token_probabilities' in row
if has_probabilities:
best_cells = get_best_cells(json.loads(row['token_probabilities']))
correct_at_k = {
k: bool(set(best_cells[:k]) & gold_coordinates)
for k in _RECALL_KS
}
else:
correct_at_k = {}
for answer_type in [AnswerType.ALL, _get_answer_type(question)]:
total_coordinates[answer_type] += len(coordinates)
total_correct[answer_type] += bool(correct_coordinates)
total_seen[answer_type] += 1
for k, correct in correct_at_k.items():
total_correct_at_k[k][answer_type] += correct
if coordinates:
sum_precision[answer_type] += correct_coordinates / len(
coordinates)
total_non_empty[answer_type] += 1
for answer_type in AnswerType:
if total[answer_type]:
recall_at_k = {
f'recall_at_{k}':
(total_correct_at_k[k][answer_type] /
total[answer_type]) if has_probabilities else None
for k in _RECALL_KS
}
yield answer_type, CellSelectionMetrics(
recall=total_correct[answer_type] / total[answer_type],
non_empty=total_non_empty[answer_type] / total[answer_type],
coverage=total_seen[answer_type] / total[answer_type],
answer_len=total_coordinates[answer_type] / total[answer_type],
precision=((sum_precision[answer_type] /
total_non_empty[answer_type])
if total_non_empty[answer_type] else None),
**recall_at_k,
)