def _evaluate_squad(self): """Runs BERT SQuAD evaluation.""" input_meta_data = self._read_input_meta_data_from_file() strategy = self._get_distribution_strategy() run_squad.predict_squad(strategy=strategy, input_meta_data=input_meta_data) dataset = self._read_predictions_dataset_from_file() predictions = self._read_predictions_from_file() eval_metrics = squad_evaluate_v1_1.evaluate(dataset, predictions) # Use F1 score as reported evaluation metric. self.eval_metrics = eval_metrics['f1']
def _evaluate_squad(self, use_ds=True): """Runs BERT SQuAD evaluation.""" assert tf.version.VERSION.startswith('2.') input_meta_data = self._read_input_meta_data_from_file() strategy = self._get_distribution_strategy(use_ds) run_squad.predict_squad(strategy=strategy, input_meta_data=input_meta_data) dataset = self._read_predictions_dataset_from_file() predictions = self._read_predictions_from_file() eval_metrics = squad_evaluate_v1_1.evaluate(dataset, predictions) # Use F1 score as reported evaluation metric. self.eval_metrics = eval_metrics['f1']
def _run_bert_squad(self): """Starts BERT SQuAD training and evaluation tasks.""" with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: input_meta_data = json.loads(reader.read().decode('utf-8')) strategy = distribution_utils.get_distribution_strategy( distribution_strategy='mirrored', num_gpus=self.num_gpus) run_squad.train_squad(strategy=strategy, input_meta_data=input_meta_data, custom_callbacks=[self.timer_callback]) run_squad.predict_squad(strategy=strategy, input_meta_data=input_meta_data) predictions_file = os.path.join(FLAGS.model_dir, 'predictions.json') eval_metrics = self._evaluate_squad(predictions_file) # Use F1 score as reported evaluation metric. self.eval_metrics = eval_metrics['f1']