def _evaluate_squad(self, predictions_file):
"""Evaluates a predictions file."""
with tf.io.gfile.GFile(SQUAD_PREDICT_FILE, 'r') as reader:
dataset_json = json.load(reader)
dataset = dataset_json['data']
with tf.io.gfile.GFile(predictions_file, 'r') as reader:
predictions = json.load(reader)
return squad_evaluate_v1_1.evaluate(dataset, predictions)
def _evaluate_squad(self):
"""Runs BERT SQuAD evaluation."""
input_meta_data = self._read_input_meta_data_from_file()
strategy = self._get_distribution_strategy()
run_squad.predict_squad(strategy=strategy, input_meta_data=input_meta_data)
dataset = self._read_predictions_dataset_from_file()
predictions = self._read_predictions_from_file()
eval_metrics = squad_evaluate_v1_1.evaluate(dataset, predictions)
# Use F1 score as reported evaluation metric.
self.eval_metrics = eval_metrics['f1']
def _evaluate_squad(self, use_ds=True):
"""Runs BERT SQuAD evaluation."""
assert tf.version.VERSION.startswith('2.')
input_meta_data = self._read_input_meta_data_from_file()
strategy = self._get_distribution_strategy(use_ds)
run_squad.predict_squad(strategy=strategy, input_meta_data=input_meta_data)
dataset = self._read_predictions_dataset_from_file()
predictions = self._read_predictions_from_file()
eval_metrics = squad_evaluate_v1_1.evaluate(dataset, predictions)
# Use F1 score as reported evaluation metric.
self.eval_metrics = eval_metrics['f1']