def evaluate_utterance_sample(sample,
model,
max_generation_length,
name="",
gold_forcing=False,
metrics=None,
total_num=-1,
database_username="",
database_password="",
database_timeout=0,
write_results=False):
"""Evaluates a sample of utterance examples.
Inputs:
sample (list of Utterance): Examples to evaluate.
model (ATISModel): Model to predict with.
max_generation_length (int): Maximum length to generate.
name (str): Name to log with.
gold_forcing (bool): Whether to force the gold tokens during decoding.
metrics (list of Metric): Metrics to evaluate with.
total_num (int): Number to divide by when reporting results.
database_username (str): Username to use for executing queries.
database_password (str): Password to use when executing queries.
database_timeout (float): Timeout on queries when executing.
write_results (bool): Whether to write the results to a file.
"""
assert metrics
if total_num < 0:
total_num = len(sample)
metrics_sums = {}
for metric in metrics:
metrics_sums[metric] = 0.
predictions_file = open(name + "_predictions.json", "w")
print("Predicting with filename " + str(name) + "_predictions.json")
progbar = get_progressbar(name, len(sample))
progbar.start()
predictions = []
for i, item in enumerate(sample):
_, loss, predicted_seq = model.eval_step(item,
max_generation_length,
feed_gold_query=gold_forcing)
loss = loss / len(item.gold_query())
predictions.append(predicted_seq)
flat_sequence = item.flatten_sequence(predicted_seq)
token_accuracy = torch_utils.per_token_accuracy(
item.gold_query(), predicted_seq)
if write_results:
write_prediction(predictions_file,
identifier=item.interaction.identifier,
input_seq=item.input_sequence(),
probability=0,
prediction=predicted_seq,
flat_prediction=flat_sequence,
gold_query=item.gold_query(),
flat_gold_queries=item.original_gold_queries(),
gold_tables=item.gold_tables(),
index_in_interaction=item.utterance_index,
database_username=database_username,
database_password=database_password,
database_timeout=database_timeout)
update_sums(metrics,
metrics_sums,
predicted_seq,
flat_sequence,
item.gold_query(),
item.original_gold_queries()[0],
gold_forcing,
loss,
token_accuracy,
database_username=database_username,
database_password=database_password,
database_timeout=database_timeout,
gold_table=item.gold_tables()[0])
progbar.update(i)
progbar.finish()
predictions_file.close()
return construct_averages(metrics_sums, total_num), None
def compute_token_accuracy_question(data):
gold_seq = data['gold_query']
pred_seq = data['flat_prediction']
return per_token_accuracy(gold_seq, pred_seq)