def predict(
self, predict_dataset, predict_examples, ignore_keys=None, metric_key_prefix: str = "test", **gen_kwargs
):
self._gen_kwargs = gen_kwargs.copy()
predict_dataloader = self.get_test_dataloader(predict_dataset)
# Temporarily disable metric computation, we will do it in the loop here.
compute_metrics = self.compute_metrics
self.compute_metrics = None
eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
try:
output = eval_loop(
predict_dataloader,
description="Prediction",
# No point gathering the predictions if there are no metrics, otherwise we defer to
# self.args.prediction_loss_only
prediction_loss_only=True if compute_metrics is None else None,
ignore_keys=ignore_keys,
)
finally:
self.compute_metrics = compute_metrics
if self.post_process_function is None or self.compute_metrics is None:
return output
predictions = self.post_process_function(predict_examples, predict_dataset, output.predictions, "predict")
metrics = self.compute_metrics(predictions)
# Prefix all keys with metric_key_prefix + '_'
for key in list(metrics.keys()):
if not key.startswith(f"{metric_key_prefix}_"):
metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)
def predict(self, predict_dataset, predict_examples, ignore_keys=None):
predict_dataloader = self.get_test_dataloader(predict_dataset)
# Temporarily disable metric computation, we will do it in the loop here.
compute_metrics = self.compute_metrics
self.compute_metrics = None
eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
try:
output = eval_loop(
predict_dataloader,
description="Prediction",
# No point gathering the predictions if there are no metrics, otherwise we defer to
# self.args.prediction_loss_only
prediction_loss_only=True if compute_metrics is None else None,
ignore_keys=ignore_keys,
)
finally:
self.compute_metrics = compute_metrics
if self.post_process_function is None or self.compute_metrics is None:
return output
predictions = self.post_process_function(predict_examples,
predict_dataset,
output.predictions, "predict")
metrics = self.compute_metrics(predictions)
return PredictionOutput(predictions=predictions.predictions,
label_ids=predictions.label_ids,
metrics=metrics)
def predict(self, test_dataset, test_examples, ignore_keys=None):
test_dataloader = self.get_test_dataloader(test_dataset)
# Temporarily disable metric computation, we will do it in the loop here.
compute_metrics = self.compute_metrics
self.compute_metrics = None
try:
output = self.prediction_loop(
test_dataloader,
description="Evaluation",
# No point gathering the predictions if there are no metrics, otherwise we defer to
# self.args.prediction_loss_only
prediction_loss_only=True if compute_metrics is None else None,
ignore_keys=ignore_keys,
)
finally:
self.compute_metrics = compute_metrics
if self.post_process_function is None or self.compute_metrics is None:
return output
# We might have removed columns from the dataset so we put them back.
if isinstance(test_dataset, datasets.Dataset):
test_dataset.set_format(type=test_dataset.format["type"],
columns=list(test_dataset.features.keys()))
eval_preds = self.post_process_function(test_examples, test_dataset,
output.predictions)
metrics = self.compute_metrics(eval_preds)
return PredictionOutput(predictions=eval_preds.predictions,
label_ids=eval_preds.label_ids,
metrics=metrics)
def predict(self, test_dataset, test_examples, ignore_keys=None):
test_dataloader = self.get_test_dataloader(test_dataset)
compute_metrics = self.compute_metrics
self.compute_metrics = None
try:
output = self.prediction_loop(
test_dataloader,
description="Evaluation",
prediction_loss_only=True if compute_metrics is None else None,
ignore_keys=ignore_keys,
)
finally:
self.compute_metrics = compute_metrics
if self.compute_metrics is None:
return output
test_dataset.set_format(type=test_dataset.format["type"],
columns=list(test_dataset.features.keys()))
eval_preds = self._post_process_function(test_examples, test_dataset,
output.predictions)
metrics = self.compute_metrics(eval_preds)
return PredictionOutput(predictions=eval_preds.predictions,
label_ids=eval_preds.label_ids,
metrics=metrics)
def predict(self,
test_dataset,
ignore_keys=None,
metric_key_prefix="test"):
"""
Run prediction and returns predictions and potential metrics
"""
if test_dataset is not None and not isinstance(test_dataset,
collections.abc.Sized):
raise ValueError("test_dataset must implement __len__")
# Test the model with the given dataloader and gather outputs
self.test_dataset = test_dataset
self.args.do_predict = True
start_time = time.time()
output = self.prediction_loop(
self.get_test_dataloader(self.test_dataset),
description="Test",
ignore_keys=ignore_keys,
metric_key_prefix=metric_key_prefix,
)
self.args.do_predict = False
# Compute answers (taking spans from original contexts)
answers_dict = utils.from_words_to_text(
test_dataset.df,
output.predictions[-2].tolist(),
output.predictions[-1].tolist(),
)
# Update metrics and patch the predictions attribute
# with the computed answers
output.metrics.update(
speed_metrics(metric_key_prefix, start_time, len(test_dataset)))
# Log final metrics to wandb
if self.wandb_callback is not None:
self.wandb_callback.save_notes(output.metrics)
return PredictionOutput(
predictions=output.predictions + (answers_dict, ),
label_ids=output.label_ids,
metrics=output.metrics,
)
def prediction_loop(self, *args, **kwargs) -> PredictionOutput:
pred_outs = super().prediction_loop(*args, **kwargs)
preds, label_ids, metrics = pred_outs.predictions, pred_outs.label_ids, pred_outs.metrics
preds = preds.squeeze()
if self.compute_metrics is not None:
metrics_no_label = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics_no_label = {}
for key in list(metrics_no_label.keys()):
if not key.startswith("eval_"):
metrics_no_label[f"eval_{key}"] = metrics_no_label.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics={
**metrics,
**metrics_no_label
})
def predict(self, test_dataset: Dataset) -> PredictionOutput:
"""
Run prediction and returns predictions and potential metrics.
Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method
will also return metrics, like in :obj:`evaluate()`.
Args:
test_dataset (:obj:`Dataset`):
Dataset to run the predictions on. If it is an :obj:`datasets.Dataset`, columns not accepted by the
``model.forward()`` method are automatically removed. Has to implement the method :obj:`__len__`
ignore_keys (:obj:`Lst[str]`, `optional`):
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions.
metric_key_prefix (:obj:`str`, `optional`, defaults to :obj:`"test"`):
An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named
"test_bleu" if the prefix is "test" (default)
.. note::
If your predictions or labels have different sequence length (for instance because you're doing dynamic
padding in a token classification task) the predictions will be padded (on the right) to allow for
concatenation into one array. The padding index is -100.
Returns: `NamedTuple` A namedtuple with the following keys:
- predictions (:obj:`np.ndarray`): The predictions on :obj:`test_dataset`.
- label_ids (:obj:`np.ndarray`, `optional`): The labels (if the dataset contained some).
- metrics (:obj:`Dict[str, float]`, `optional`): The potential dictionary of metrics (if the dataset
contained labels).
"""
test_dataloader = self.get_test_dataloader(test_dataset)
output = self._prediction_loop(test_dataloader,
description="Prediction")
self.log(output.metrics)
return PredictionOutput(predictions=output.predictions,
label_ids=output.label_ids,
metrics=output.metrics)
def prediction_loop(
self, dataloader: DataLoader, description: str, prediction_loss_only: Optional[bool] = None
) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.
Works both with or without labels.
"""
if hasattr(self, "_prediction_loop"):
warnings.warn(
"The `_prediction_loop` method is deprecated and won't be called in a future version, define `prediction_loop` in your subclass.",
FutureWarning,
)
return self._prediction_loop(dataloader, description, prediction_loss_only=prediction_loss_only)
prediction_loss_only = (
prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
)
'''
assert not getattr(
self.model.config, "output_attentions", False
), "The prediction loop does not work with `output_attentions=True`."
assert not getattr(
self.model.config, "output_hidden_states", False
), "The prediction loop does not work with `output_hidden_states=True`."
'''
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
else:
model = self.model
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
'''
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
'''
eval_losses: List[float] = []
preds: torch.Tensor = None
label_ids: torch.Tensor = None
model.eval()
if is_torch_tpu_available():
dataloader = pl.ParallelLoader(dataloader, [self.args.device]).per_device_loader(self.args.device)
if self.args.past_index >= 0:
self._past = None
disable_tqdm = not self.is_local_process_zero() or self.args.disable_tqdm
for inputs in tqdm(dataloader, desc=description, disable=disable_tqdm):
loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only)
batch_size = inputs[list(inputs.keys())[0]].shape[0]
if loss is not None:
eval_losses.extend([loss] * batch_size)
if logits is not None:
preds = logits if preds is None else nested_concat(preds, logits, dim=0)
if labels is not None:
label_ids = labels if label_ids is None else nested_concat(label_ids, labels, dim=0)
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of the evaluation loop
delattr(self, "_past")
if self.args.local_rank != -1:
# In distributed mode, concatenate all results from all nodes:
if preds is not None:
preds = distributed_concat(preds, num_total_examples=self.num_examples(dataloader))
if label_ids is not None:
label_ids = distributed_concat(label_ids, num_total_examples=self.num_examples(dataloader))
elif is_torch_tpu_available():
# tpu-comment: Get all predictions and labels from all worker shards of eval dataset
if preds is not None:
preds = nested_xla_mesh_reduce(preds, "eval_preds")
if label_ids is not None:
label_ids = nested_xla_mesh_reduce(label_ids, "eval_label_ids")
if eval_losses is not None:
eval_losses = xm.mesh_reduce("eval_losses", torch.tensor(eval_losses), torch.cat).tolist()
# Finally, turn the aggregated tensors into numpy arrays.
if preds is not None:
preds = nested_numpify(preds)
if label_ids is not None:
label_ids = nested_numpify(label_ids)
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
if self.args.local_rank != -1:
metrics["eval_loss"] = (
distributed_broadcast_scalars(eval_losses, num_total_examples=self.num_examples(dataloader))
.mean()
.item()
)
else:
metrics["eval_loss"] = np.mean(eval_losses)
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
def prediction_loop(self, data_loader, world_size):
num_examples = len(data_loader.dataset)
batch_size = data_loader.batch_size
eval_losses_gatherer = DistributedTensorGatherer(
world_size, num_examples, make_multiple_of=batch_size)
preds_gatherer = DistributedTensorGatherer(world_size, num_examples)
labels_gatherer = DistributedTensorGatherer(world_size, num_examples)
losses_host, preds_host, labels_host = None, None, None
self.model.eval()
for step, inputs in enumerate(data_loader):
loss, logits, labels = self.prediction_step(inputs)
losses = loss.repeat(batch_size)
losses_host = losses if losses_host is None else torch.cat(
(losses_host, losses), dim=0)
preds_host = logits if preds_host is None else trainer_pt_utils.nested_concat(
preds_host, logits, padding_index=-100)
labels_host = labels if labels_host is None else trainer_pt_utils.nested_concat(
labels_host, labels, padding_index=-100)
eval_losses_gatherer.add_arrays(
trainer_pt_utils.nested_numpify(losses_host))
preds_gatherer.add_arrays(
trainer_pt_utils.nested_numpify(preds_host))
labels_gatherer.add_arrays(
trainer_pt_utils.nested_numpify(labels_host))
losses_host, preds_host, labels_host = None, None, None
eval_loss = eval_losses_gatherer.finalize()
preds = preds_gatherer.finalize()
labels_ids = labels_gatherer.finalize()
if self.type_score == "PER":
preds_ids = np.argmax(preds, axis=-1)
predicted_phonemes = self.processor.batch_decode(
torch.from_numpy(preds_ids))
true_phonemes = self.processor.batch_decode(
torch.from_numpy(labels_ids))
per = generate_per_score(true_phonemes, predicted_phonemes)
return per
elif self.type_score == "WER":
pred = EvalPrediction(predictions=preds, label_ids=labels_ids)
pred_logits = pred.predictions
pred_ids = np.argmax(pred_logits, axis=-1)
pred.label_ids[pred.label_ids ==
-100] = self.processor.tokenizer.pad_token_id
pred_str = self.processor.batch_decode(pred_ids)
# we do not want to group tokens when computing the metrics
label_str = self.processor.batch_decode(pred.label_ids,
group_tokens=False)
metrics = compute_wer(pred_str, label_str)
metrics = denumpify_detensorize(metrics)
metrics["t_loss"] = eval_loss.mean().item()
wer = PredictionOutput(preds, labels_ids, metrics).metrics["wer"]
return wer
def _prediction_loop(
self,
dataset: tf.data.Dataset,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
logger.info("***** Running %s *****", description)
logger.info(" Batch size = %d", self.args.eval_batch_size)
label_ids: np.ndarray = None
preds: np.ndarray = None
step: int = 1
for features, labels in dataset:
step = tf.convert_to_tensor(step, dtype=tf.int64)
loss, logits = self._evaluate_steps(features, labels)
loss = tf.reduce_mean(loss)
if not prediction_loss_only:
if isinstance(logits, tuple):
logits = logits[0]
if isinstance(labels, tuple):
labels = labels[0]
if self.args.n_gpu > 1:
for val in logits.values:
if preds is None:
preds = val.numpy()
else:
preds = np.append(preds, val.numpy(), axis=0)
for val in labels.values:
if label_ids is None:
label_ids = val.numpy()
else:
label_ids = np.append(label_ids,
val.numpy(),
axis=0)
else:
if preds is None:
preds = logits.numpy()
else:
preds = np.append(preds, logits.numpy(), axis=0)
if label_ids is None:
label_ids = labels.numpy()
else:
label_ids = np.append(label_ids,
labels.numpy(),
axis=0)
step += 1
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
metrics["eval_loss"] = loss.numpy()
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)
def _prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
else:
model = self.model
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
logger.info(" Decode mode = %s", self.args.decode_mode)
eval_losses: List[float] = []
model.eval()
metric = ParsingMetric()
if is_tpu_available():
dataloader = pl.ParallelLoader(
dataloader,
[self.args.device]).per_device_loader(self.args.device)
for inputs in tqdm(dataloader, desc=description):
for k, v in inputs.items():
inputs[k] = v.to(self.args.device)
with torch.no_grad():
step_eval_loss, rel_preds, arc_preds = model(
**inputs, adapter_names=self.adapter_names)
eval_losses += [step_eval_loss.mean().item()]
mask = inputs["labels_arcs"].ne(self.model.config.pad_token_id)
predictions_arcs = torch.argmax(arc_preds, dim=-1)[mask]
labels_arcs = inputs["labels_arcs"][mask]
predictions_rels, labels_rels = rel_preds[mask], inputs[
"labels_rels"][mask]
predictions_rels = predictions_rels[torch.arange(len(labels_arcs)),
labels_arcs]
predictions_rels = torch.argmax(predictions_rels, dim=-1)
metric.add(labels_arcs, labels_rels, predictions_arcs,
predictions_rels)
results = metric.get_metric()
results[f"{description}_loss"] = np.mean(eval_losses)
# Add predictions_rels to output, even though we are only interested in the metrics
return PredictionOutput(predictions=predictions_rels,
label_ids=None,
metrics=results)
def prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None,
ignore_keys: Optional[List[str]] = None,
metric_key_prefix: str = "eval",
) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.
Works both with or without labels.
"""
if not isinstance(dataloader.dataset, collections.abc.Sized):
raise ValueError("dataset must implement __len__")
prediction_loss_only = (
prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
)
## added to extract the target model for the task
model = self.model.taskmodels_dict[self.eval_task_name]
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
batch_size = dataloader.batch_size
num_examples = self.num_examples(dataloader)
# logger.info("***** Running %s *****", description)
# logger.info(" Num examples = %d", num_examples)
# logger.info(" Batch size = %d", batch_size)
losses_host: torch.Tensor = None
preds_host: Union[torch.Tensor, List[torch.Tensor]] = None
labels_host: Union[torch.Tensor, List[torch.Tensor]] = None
world_size = 1
if is_torch_tpu_available():
world_size = xm.xrt_world_size()
elif self.args.local_rank != -1:
world_size = torch.distributed.get_world_size()
world_size = max(1, world_size)
eval_losses_gatherer = DistributedTensorGatherer(world_size, num_examples, make_multiple_of=batch_size)
if not prediction_loss_only:
preds_gatherer = DistributedTensorGatherer(world_size, num_examples)
labels_gatherer = DistributedTensorGatherer(world_size, num_examples)
model.eval()
if is_torch_tpu_available():
dataloader = pl.ParallelLoader(dataloader, [self.args.device]).per_device_loader(self.args.device)
if self.args.past_index >= 0:
self._past = None
self.callback_handler.eval_dataloader = dataloader
for step, inputs in enumerate(dataloader):
# added to reshape teh list of dictionaries of tensors into a single dictionary of tensors
inputs = self.reshape_inputs(inputs)
loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only,
ignore_keys=ignore_keys)
if loss is not None:
losses = loss.repeat(batch_size)
losses_host = losses if losses_host is None else torch.cat((losses_host, losses), dim=0)
if logits is not None:
preds_host = logits if preds_host is None else nested_concat(preds_host, logits, padding_index=-100)
if labels is not None:
labels_host = labels if labels_host is None else nested_concat(labels_host, labels,
padding_index=-100)
self.control = self.callback_handler.on_prediction_step(self.args, self.state, self.control)
# Gather all tensors and put them back on the CPU if we have done enough accumulation steps.
if self.args.eval_accumulation_steps is not None and (
step + 1) % self.args.eval_accumulation_steps == 0:
eval_losses_gatherer.add_arrays(self._gather_and_numpify(losses_host, "eval_losses"))
if not prediction_loss_only:
preds_gatherer.add_arrays(self._gather_and_numpify(preds_host, "eval_preds"))
labels_gatherer.add_arrays(self._gather_and_numpify(labels_host, "eval_label_ids"))
# Set back to None to begin a new accumulation
losses_host, preds_host, labels_host = None, None, None
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of the evaluation loop
delattr(self, "_past")
# Gather all remaining tensors and put them back on the CPU
eval_losses_gatherer.add_arrays(self._gather_and_numpify(losses_host, "eval_losses"))
if not prediction_loss_only:
preds_gatherer.add_arrays(self._gather_and_numpify(preds_host, "eval_preds"))
labels_gatherer.add_arrays(self._gather_and_numpify(labels_host, "eval_label_ids"))
eval_loss = eval_losses_gatherer.finalize()
preds = preds_gatherer.finalize() if not prediction_loss_only else None
label_ids = labels_gatherer.finalize() if not prediction_loss_only else None
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if eval_loss is not None:
metrics[f"{metric_key_prefix}_loss"] = eval_loss.mean().item()
# Prefix all keys with metric_key_prefix + '_'
for key in list(metrics.keys()):
if not key.startswith(f"{metric_key_prefix}_"):
metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
def prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None,
ignore_keys: Optional[List[str]] = None,
metric_key_prefix: str = "eval",
) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.
Works both with or without labels.
"""
if not isinstance(dataloader.dataset, collections.abc.Sized):
raise ValueError("dataset must implement __len__")
prediction_loss_only = (
prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
)
if self.args.deepspeed and not self.args.do_train:
# no harm, but flagging to the user that deepspeed config is ignored for eval
# flagging only for when --do_train wasn't passed as only then it's redundant
logger.info("Detected the deepspeed argument but it will not be used for evaluation")
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways
# if full fp16 is wanted on eval and this ``evaluation`` or ``predict`` isn't called while
# ``train`` is running, half it first and then put on device
batch_size = dataloader.batch_size
num_examples = self.num_examples(dataloader)
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", num_examples)
logger.info(" Batch size = %d", batch_size)
losses_host: torch.Tensor = None
preds_host: Union[torch.Tensor, List[torch.Tensor]] = None
labels_host: Union[torch.Tensor, List[torch.Tensor]] = None
gumbel_host: Union[torch.Tensor, List[torch.Tensor]] = None
sentence_labels_host: Union[torch.Tensor, List[torch.Tensor]] = None
sentence_indicator_host: Union[torch.Tensor, List[torch.Tensor]] = None
world_size = 1
if is_torch_tpu_available():
world_size = xm.xrt_world_size()
elif self.args.local_rank != -1:
world_size = dist.get_world_size()
world_size = max(1, world_size)
eval_losses_gatherer = DistributedTensorGatherer(world_size, num_examples, make_multiple_of=batch_size)
if not prediction_loss_only:
preds_gatherer = DistributedTensorGatherer(world_size, num_examples)
labels_gatherer = DistributedTensorGatherer(world_size, num_examples)
gumbel_gatherer = DistributedTensorGatherer(world_size, num_examples)
sentence_labels_gatherer = DistributedTensorGatherer(world_size, num_examples)
sentence_indicator_gatherer = DistributedTensorGatherer(world_size, num_examples)
model.eval()
if self.args.past_index >= 0:
self._past = None
self.callback_handler.eval_dataloader = dataloader
for step, inputs in enumerate(dataloader):
loss, logits, labels, gumbel_output, sentence_labels, sentence_indicator = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)
if loss is not None:
losses = loss.repeat(batch_size)
losses_host = losses if losses_host is None else torch.cat((losses_host, losses), dim=0)
if logits is not None:
preds_host = logits if preds_host is None else nested_concat(preds_host, logits, padding_index=-100)
if labels is not None:
labels_host = labels if labels_host is None else nested_concat(labels_host, labels, padding_index=-100)
if gumbel_output is not None:
gumbel_host = gumbel_output if gumbel_host is None else nested_concat(gumbel_host, gumbel_output, padding_index=-1)
if sentence_labels is not None:
sentence_labels_host = sentence_labels if sentence_labels_host is None else nested_concat(sentence_labels_host, sentence_labels, padding_index=-1)
if sentence_indicator is not None:
sentence_indicator_host = sentence_indicator if sentence_indicator_host is None else nested_concat(sentence_indicator_host, sentence_indicator, padding_index=-100)
self.control = self.callback_handler.on_prediction_step(self.args, self.state, self.control)
# Gather all tensors and put them back on the CPU if we have done enough accumulation steps.
if self.args.eval_accumulation_steps is not None and (step + 1) % self.args.eval_accumulation_steps == 0:
eval_losses_gatherer.add_arrays(self._gather_and_numpify(losses_host, "eval_losses"))
if not prediction_loss_only:
preds_gatherer.add_arrays(self._gather_and_numpify(preds_host, "eval_preds"))
labels_gatherer.add_arrays(self._gather_and_numpify(labels_host, "eval_label_ids"))
gumbel_gatherer.add_arrays(self._gather_and_numpify(gumbel_host, "eval_gumbel_output"))
sentence_labels_gatherer.add_arrays(self._gather_and_numpify(sentence_labels_host, "eval_sentence_idxs"))
sentence_indicator_gatherer.add_arrays(self._gather_and_numpify(sentence_indicator_host, "eval_sentence_indicator"))
# Set back to None to begin a new accumulation
losses_host, preds_host, labels_host, gumbel_host, sentence_labels_host, sentence_indicator_host = None, None, None, None, None, None
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of the evaluation loop
delattr(self, "_past")
# Gather all remaining tensors and put them back on the CPU
eval_losses_gatherer.add_arrays(self._gather_and_numpify(losses_host, "eval_losses"))
if not prediction_loss_only:
preds_gatherer.add_arrays(self._gather_and_numpify(preds_host, "eval_preds"))
labels_gatherer.add_arrays(self._gather_and_numpify(labels_host, "eval_label_ids"))
gumbel_gatherer.add_arrays(self._gather_and_numpify(gumbel_host, "eval_gumbel_output"))
sentence_labels_gatherer.add_arrays(self._gather_and_numpify(sentence_labels_host, "eval_sentence_idxs"))
sentence_indicator_gatherer.add_arrays(self._gather_and_numpify(sentence_indicator_host, "eval_sentence_indicator"))
eval_loss = eval_losses_gatherer.finalize()
preds = preds_gatherer.finalize() if not prediction_loss_only else None
label_ids = labels_gatherer.finalize() if not prediction_loss_only else None
gumbel_outputs = gumbel_gatherer.finalize() if not prediction_loss_only else None
sentence_idxs = sentence_labels_gatherer.finalize() if not prediction_loss_only else None
sentence_indicators = sentence_indicator_gatherer.finalize() if not prediction_loss_only else None
print(sentence_idxs, 'test')
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(preds, label_ids, gumbel_outputs, sentence_idxs, sentence_indicators)
else:
metrics = {}
if eval_loss is not None:
metrics[f"{metric_key_prefix}_loss"] = eval_loss.mean().item()
# Prefix all keys with metric_key_prefix + '_'
for key in list(metrics.keys()):
if not key.startswith(f"{metric_key_prefix}_"):
metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
def prediction_loop(
self,
dataset: tf.data.Dataset,
steps: int,
num_examples: int,
description: str,
prediction_loss_only: Optional[bool] = None,
) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :func:`~transformers.TFTrainer.evaluate` and
:func:`~transformers.TFTrainer.predict`.
Works both with or without labels.
"""
prediction_loss_only = (
prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
)
logger.info("***** Running %s *****", description)
logger.info(" Num examples in dataset = %d", num_examples)
if description == "Evaluation":
logger.info(" Num examples in used in evaluation = %d", self.args.eval_batch_size * steps)
logger.info(" Batch size = %d", self.args.eval_batch_size)
label_ids: np.ndarray = None
preds: np.ndarray = None
self.eval_loss.reset_states()
# Reset the past mems state at the beginning of the evaluation if necessary.
if self.args.past_index >= 0:
self._past = None
for step, batch in enumerate(dataset):
logits = self.distributed_prediction_steps(batch)
_, labels = batch
if not prediction_loss_only:
if isinstance(logits, tuple):
logits = logits[0]
if isinstance(labels, tuple):
labels = labels[0]
if self.args.n_replicas > 1:
for val in logits.values:
if preds is None:
preds = val.numpy()
else:
preds = np.append(preds, val.numpy(), axis=0)
for val in labels.values:
if label_ids is None:
label_ids = val.numpy()
else:
label_ids = np.append(label_ids, val.numpy(), axis=0)
else:
if preds is None:
preds = logits.numpy()
else:
preds = np.append(preds, logits.numpy(), axis=0)
if label_ids is None:
label_ids = labels.numpy()
else:
label_ids = np.append(label_ids, labels.numpy(), axis=0)
if step == steps - 1:
break
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
metrics["eval_loss"] = self.eval_loss.result().numpy() / steps
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of training
delattr(self, "_past")
return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
def _prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
model = self.model
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
eval_losses: List[float] = []
preds: torch.Tensor = None
label_ids: torch.Tensor = None
model.eval()
for inputs in tqdm(dataloader, desc=description):
has_labels = any(
inputs.get(k) is not None
for k in ["labels", "lm_labels", "masked_lm_labels"])
for k, v in inputs.items():
inputs[k] = v.to(self.device)
with torch.no_grad():
outputs = model(**inputs)
if has_labels:
step_eval_loss, logits = outputs[:2]
eval_losses += [step_eval_loss.mean().item()]
else:
logits = outputs[0]
if not prediction_loss_only:
if preds is None:
preds = logits.detach()
else:
preds = torch.cat((preds, logits.detach()), dim=0)
if inputs.get("labels") is not None:
if label_ids is None:
label_ids = inputs["labels"].detach()
else:
label_ids = torch.cat(
(label_ids, inputs["labels"].detach()), dim=0)
if self.args.local_rank != -1:
# In distributed mode, concatenate all results from all nodes:
if preds is not None:
preds = self.distributed_concat(
preds, num_total_examples=self.num_examples(dataloader))
if label_ids is not None:
label_ids = self.distributed_concat(
label_ids,
num_total_examples=self.num_examples(dataloader))
if preds is not None:
preds = preds.cpu().numpy()
if label_ids is not None:
label_ids = label_ids.cpu().numpy()
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["eval_loss"] = np.mean(eval_losses)
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)
def prediction_loop(self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None,
extract_path: Optional[str] = None,
cache_path: Optional[str] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.
Works both with or without labels.
"""
prediction_loss_only = (prediction_loss_only
if prediction_loss_only is not None else
self.args.prediction_loss_only)
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
else:
model = self.model
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
batch_size = dataloader.batch_size
eval_losses: List[float] = []
hidden_states: torch.tensor = None
preds: torch.Tensor = None
label_ids: torch.Tensor = None
model.eval()
if self.args.past_index >= 0:
self._past = None
# Unfortunate, but we'll run through the dataloader once to count the number of tokens (or this could be pre-processed)
if extract_path is not None:
stimulus_mask = lambda tokens: (tokens != 101) & (tokens != 102
) & (tokens != 0)
cached_masks = None
if osp.exists(f"{cache_path}.npy"):
# np instead of torch, something's funky with Vivek's env.
cached_masks = torch.from_numpy(np.load(f"{cache_path}.npy"))
else:
all_masks = None
limit_tokens = self.custom_cfg.TASK.EXTRACT_TOKENS_LIMIT
# Calculate the random ratio of tokens to grab (we specify number of tokens to extract)
total_tokens = 0
for inputs in dataloader:
tokens = inputs["input_ids"]
total_tokens += stimulus_mask(tokens).sum()
subset_ratio = torch.true_divide(limit_tokens, total_tokens)
# Seed, we want to be sure that we're finding the same stimuli
disable_tqdm = not self.is_local_process_zero(
) or self.args.disable_tqdm
samples_count = 0
for inputs in tqdm(dataloader, desc=description, disable=disable_tqdm):
loss, logits, labels, states = self.prediction_step(
model,
inputs,
prediction_loss_only,
output_hidden_states=extract_path is not None)
batch_size = inputs[list(inputs.keys())[0]].shape[0]
if loss is not None:
eval_losses.append(loss * batch_size)
if states is not None:
# L + 1 [ Batch x Length x Hidden ] (layers and embedding)
if cached_masks is not None:
cached_masks = cached_masks.to(logits.device)
mask = cached_masks[samples_count:samples_count +
inputs["input_ids"].shape[0]] # B x T
mask = mask[:, :inputs["input_ids"].
shape[1]] # Dynamic padding
else:
subset_mask = torch.full(inputs["input_ids"].shape,
subset_ratio,
device=logits.device)
mask = (torch.bernoulli(subset_mask).long() &
stimulus_mask(inputs["input_ids"])).bool() # B X T
if all_masks is None:
all_masks = mask
else:
all_masks = nested_concat(all_masks,
mask,
padding_index=-100) # B x T
# [1:] to drop embedding layer
states = torch.stack(states)[1:].permute(1, 2, 0,
3) # B x T x L x H
target_tokens = states[mask] # M x L x H
if hidden_states is None:
hidden_states = target_tokens
else:
hidden_states = torch.cat([hidden_states, target_tokens],
dim=0)
samples_count += batch_size
if logits is not None:
preds = logits if preds is None else nested_concat(
preds, logits, padding_index=-100)
if labels is not None:
label_ids = labels if label_ids is None else nested_concat(
label_ids, labels, padding_index=-100)
if extract_path is not None:
os.makedirs(osp.split(extract_path)[0], exist_ok=True)
np.save(extract_path,
hidden_states.half().cpu().numpy()) # half to save memory
if cached_masks is None:
os.makedirs(osp.split(cache_path)[0], exist_ok=True)
np.save(cache_path, all_masks.cpu().numpy())
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of the evaluation loop
delattr(self, "_past")
if self.args.local_rank != -1:
# In distributed mode, concatenate all results from all nodes:
if preds is not None:
preds = self.distributed_concat(
preds, num_total_examples=self.num_examples(dataloader))
if label_ids is not None:
label_ids = self.distributed_concat(
label_ids,
num_total_examples=self.num_examples(dataloader))
# Finally, turn the aggregated tensors into numpy arrays.
if preds is not None:
preds = preds.cpu().numpy()
if label_ids is not None:
label_ids = label_ids.cpu().numpy()
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["eval_loss"] = np.sum(eval_losses) / samples_count
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)
def _prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
# multi-gpu eval
if self.args.n_gpu > 1 and not isinstance(self.model,
torch.nn.DataParallel):
model = torch.nn.DataParallel(self.model)
else:
model = self.model
model.to(self.args.device)
if is_tpu_available():
batch_size = dataloader._loader._loader.batch_size
else:
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
eval_losses: List[float] = []
preds_t1: np.ndarray = None
preds_t2: np.ndarray = None
label_ids_t1: np.ndarray = None
label_ids_t2: np.ndarray = None
model.eval()
for inputs in tqdm(dataloader, desc=description):
has_labels = any(
inputs.get(k) is not None for k in [
"labels", "labels_t1", "labels_t2", "lm_labels",
"masked_lm_labels"
])
for k, v in inputs.items():
inputs[k] = v.to(self.args.device)
with torch.no_grad():
outputs = model(**inputs)
if has_labels:
if self.alternate:
step_eval_loss, logits, task = outputs[:3]
else:
step_eval_loss, logits_t1, logits_t2 = outputs[:3]
eval_losses += [step_eval_loss.mean().item()]
else:
logits = outputs[0]
if self.alternate:
if not prediction_loss_only:
if task == 0:
if preds_t1 is None:
preds_t1 = logits.detach().cpu().numpy()
else:
preds_t1 = np.append(preds_t1,
logits.detach().cpu().numpy(),
axis=0)
if inputs.get("labels") is not None:
if label_ids_t1 is None:
label_ids_t1 = inputs["labels"].detach().cpu(
).numpy()
else:
label_ids_t1 = np.append(
label_ids_t1,
inputs["labels"].detach().cpu().numpy(),
axis=0)
elif task == 1:
if preds_t2 is None:
preds_t2 = logits.detach().cpu().numpy()
else:
preds_t2 = np.append(preds_t2,
logits.detach().cpu().numpy(),
axis=0)
if inputs.get("labels") is not None:
if label_ids_t2 is None:
label_ids_t2 = inputs["labels"].detach().cpu(
).numpy()
else:
label_ids_t2 = np.append(
label_ids_t2,
inputs["labels"].detach().cpu().numpy(),
axis=0)
else:
if not prediction_loss_only:
if preds_t1 is None or preds_t2 is None:
preds_t1 = logits_t1.detach().cpu().numpy()
preds_t2 = logits_t1.detach().cpu().numpy()
else:
preds_t1 = np.append(preds_t1,
logits_t1.detach().cpu().numpy(),
axis=0)
preds_t2 = np.append(preds_t2,
logits_t2.detach().cpu().numpy(),
axis=0)
if inputs.get("labels_t1") is not None:
if label_ids_t1 is None or label_ids_t2 is None:
label_ids_t1 = inputs["labels_t1"].detach().cpu(
).numpy()
label_ids_t2 = inputs["labels_t2"].detach().cpu(
).numpy()
else:
label_ids_t1 = np.append(
label_ids_t1,
inputs["labels_t1"].detach().cpu().numpy(),
axis=0)
label_ids_t2 = np.append(
label_ids_t2,
inputs["labels_t2"].detach().cpu().numpy(),
axis=0)
# if is_tpu_available() and preds is not None and label_ids is not None:
# # tpu-comment: Get all predictions and labels from all worker shards of eval dataset
# preds = xm.mesh_reduce("eval_preds", preds, np.concatenate)
# label_ids = xm.mesh_reduce("eval_out_label_ids", label_ids, np.concatenate)
metrics = {}
if self.compute_metrics is not None:
if preds_t1 is not None and label_ids_t1 is not None:
metrics["task 1"] = self.compute_metrics(
EvalPrediction(predictions=preds_t1,
label_ids=label_ids_t1))
if preds_t2 is not None and label_ids_t2 is not None:
metrics["task 2"] = self.compute_metrics(
EvalPrediction(predictions=preds_t2,
label_ids=label_ids_t2))
if len(eval_losses) > 0:
metrics["eval_loss"] = np.mean(eval_losses)
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return (PredictionOutput(predictions=preds_t1,
label_ids=label_ids_t1,
metrics=metrics),
PredictionOutput(predictions=preds_t2,
label_ids=label_ids_t2,
metrics=metrics))
def _prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
else:
model = self.model
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
eval_losses: List[float] = []
preds: torch.Tensor = None
label_ids: torch.Tensor = None
model.eval()
if is_torch_tpu_available():
dataloader = pl.ParallelLoader(
dataloader,
[self.args.device]).per_device_loader(self.args.device)
for inputs in tqdm(dataloader, desc=description):
has_labels = any(
inputs.get(k) is not None
for k in ["labels", "lm_labels", "masked_lm_labels"])
for k, v in inputs.items():
inputs[k] = v.to(self.args.device)
with torch.no_grad():
outputs = model(**inputs)
if has_labels:
step_eval_loss, logits = outputs[:2]
eval_losses += [step_eval_loss.mean().item()]
else:
logits = outputs[0]
if not prediction_loss_only:
if preds is None:
preds = logits.detach()
else:
preds = torch.cat((preds, logits.detach()), dim=0)
if inputs.get("labels") is not None:
if label_ids is None:
label_ids = inputs["labels"].detach()
else:
label_ids = torch.cat(
(label_ids, inputs["labels"].detach()), dim=0)
if self.args.local_rank != -1:
# In distributed mode, concatenate all results from all nodes:
if preds is not None:
preds = self.distributed_concat(
preds, num_total_examples=self.num_examples(dataloader))
if label_ids is not None:
label_ids = self.distributed_concat(
label_ids,
num_total_examples=self.num_examples(dataloader))
elif is_torch_tpu_available():
# tpu-comment: Get all predictions and labels from all worker shards of eval dataset
if preds is not None:
preds = xm.mesh_reduce("eval_preds", preds, torch.cat)
if label_ids is not None:
label_ids = xm.mesh_reduce("eval_label_ids", label_ids,
torch.cat)
# Finally, turn the aggregated tensors into numpy arrays.
if preds is not None:
preds = preds.cpu().numpy()
if label_ids is not None:
label_ids = label_ids.cpu().numpy()
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["eval_loss"] = np.mean(eval_losses)
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)
def _prediction_loop(
self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
# multi-gpu eval
if self.args.n_gpu > 1 and not isinstance(self.model,
torch.nn.DataParallel):
model = torch.nn.DataParallel(self.model)
else:
model = self.model
model.to(self.args.device)
if is_tpu_available():
batch_size = dataloader._loader._loader.batch_size
else:
batch_size = dataloader.batch_size
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", self.num_examples(dataloader))
logger.info(" Batch size = %d", batch_size)
eval_losses: List[float] = []
eval_tag_losses = []
eval_gen_losses = []
eval_cov_losses = []
preds = []
label_ids = []
model.eval()
for inputs in tqdm(dataloader, desc=description):
for k, v in inputs.items():
inputs[k] = v.to(self.args.device)
with torch.no_grad():
outputs = model(**inputs)
step_eval_loss, logits = outputs[:2]
other_loss = outputs[-1]
eval_losses += [step_eval_loss.mean().item()]
eval_tag_losses += [other_loss['tag_loss'].mean().item()]
eval_gen_losses += [other_loss['gen_loss'].mean().item()]
eval_cov_losses += [other_loss['cov_loss'].mean().item()]
if not prediction_loss_only:
preds.append(logits.detach().cpu().numpy().argmax(-1))
if inputs.get("tgt_token") is not None:
label_ids.append(
inputs["tgt_token"][:, 1:].detach().cpu().numpy())
if is_tpu_available():
# tpu-comment: Get all predictions and labels from all worker shards of eval dataset
preds = xm.mesh_reduce("eval_preds", preds, np.concatenate)
label_ids = xm.mesh_reduce("eval_out_label_ids", label_ids,
np.concatenate)
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["eval_loss"] = np.mean(eval_losses)
if len(eval_tag_losses) > 0:
metrics["eval_tag_loss"] = np.mean(eval_tag_losses)
if len(eval_gen_losses) > 0:
metrics["eval_gen_loss"] = np.mean(eval_gen_losses)
if len(eval_cov_losses) > 0:
metrics["eval_cov_loss"] = np.mean(eval_cov_losses)
if metrics["eval_cov_loss"] != 0:
metrics["eval_loss"] = metrics["eval_tag_loss"] + metrics[
"eval_gen_loss"]
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)