def batch_predict(
self,
dataset: Dataset,
dataset_name: str = None,
):
with dataset.initialize_batcher(self._batch_size,
should_shuffle=False,
horovod=self._horovod) as batcher:
progress_bar = None
if self.is_coordinator():
progress_bar = tqdm(
desc="Prediction" if dataset_name is None else
f"Prediction {dataset_name: <5.5}",
total=batcher.steps_per_epoch,
file=sys.stdout,
disable=is_progressbar_disabled(),
)
predictions = defaultdict(list)
while not batcher.last_batch():
batch = batcher.next_batch()
preds = self._predict(self.model, batch)
self._accumulate_preds(preds, predictions)
if self.is_coordinator():
progress_bar.update(1)
if self.is_coordinator():
progress_bar.close()
# consolidate predictions from each batch to a single tensor
self._concat_preds(predictions)
return from_numpy_dataset(predictions)
def batch_evaluation(self, dataset, collect_predictions=False, dataset_name=None):
with dataset.initialize_batcher(self._batch_size, should_shuffle=False, horovod=self._horovod) as batcher:
progress_bar = None
if self.is_coordinator():
progress_bar = tqdm(
desc="Evaluation" if dataset_name is None else f"Evaluation {dataset_name: <5.5}",
total=batcher.steps_per_epoch,
file=sys.stdout,
disable=is_progressbar_disabled(),
)
predictions = defaultdict(list)
while not batcher.last_batch():
batch = batcher.next_batch()
logger.debug(
f"evaluation for {dataset_name}: obtained next batch "
f"memory used: {psutil.Process(os.getpid()).memory_info()[0] / 1e6:0.2f}MB"
)
inputs = {
i_feat.feature_name: torch.from_numpy(batch[i_feat.proc_column]).to(self.device)
for i_feat in self.model.input_features.values()
}
targets = {
o_feat.feature_name: torch.from_numpy(batch[o_feat.proc_column]).to(self.device)
for o_feat in self.model.output_features.values()
}
preds = self.model.evaluation_step(inputs, targets)
# accumulate predictions from batch for each output feature
if collect_predictions:
for of_name, of_preds in preds.items():
for pred_name, pred_values in of_preds.items():
if pred_name not in EXCLUDE_PRED_SET:
key = f"{of_name}_{pred_name}"
predictions[key].append(pred_values)
if self.is_coordinator():
progress_bar.update(1)
logger.debug(
f"evaluation for {dataset_name}: completed batch {progress_bar.n} "
f"memory used: {psutil.Process(os.getpid()).memory_info()[0] / 1e6:0.2f}MB"
)
if self.is_coordinator():
progress_bar.close()
# consolidate predictions from each batch to a single tensor
if collect_predictions:
for key, pred_value_list in predictions.items():
predictions[key] = torch.cat(pred_value_list, dim=0).clone().detach().cpu().numpy()
metrics = self.model.get_metrics()
metrics = self.merge_workers_metrics(metrics)
self.model.reset_metrics()
return metrics, from_numpy_dataset(predictions)
def batch_evaluation(self,
model,
dataset,
collect_predictions=False,
dataset_name=None):
with dataset.initialize_batcher(self._batch_size,
should_shuffle=False,
horovod=self._horovod) as batcher:
progress_bar = None
if self.is_coordinator():
progress_bar = tqdm(
desc='Evaluation' if dataset_name is None else
'Evaluation {0: <5.5}'.format(dataset_name),
total=batcher.steps_per_epoch,
file=sys.stdout,
disable=is_progressbar_disabled())
predictions = defaultdict(list)
while not batcher.last_batch():
batch = batcher.next_batch()
inputs = {
i_feat.feature_name: batch[i_feat.proc_column]
for i_feat in model.input_features.values()
}
targets = {
o_feat.feature_name: batch[o_feat.proc_column]
for o_feat in model.output_features.values()
}
preds = model.evaluation_step(inputs, targets)
# accumulate predictions from batch for each output feature
if collect_predictions:
for of_name, of_preds in preds.items():
for pred_name, pred_values in of_preds.items():
if pred_name not in EXCLUE_PRED_SET:
key = f'{of_name}_{pred_name}'
predictions[key].append(pred_values)
if self.is_coordinator():
progress_bar.update(1)
if self.is_coordinator():
progress_bar.close()
# consolidate predictions from each batch to a single tensor
if collect_predictions:
for key, pred_value_list in predictions.items():
predictions[key] = tf.concat(pred_value_list, axis=0).numpy()
metrics = model.get_metrics()
metrics = self.merge_workers_metrics(metrics)
model.reset_metrics()
return metrics, from_numpy_dataset(predictions)
def predict_single(self, batch):
prev_model_training_mode = self.model.training # store previous model training mode
self.model.eval() # set model to eval mode
with torch.no_grad():
predictions = defaultdict(list)
preds = self._predict(self.model, batch)
self._accumulate_preds(preds, predictions)
self._concat_preds(predictions)
# reset model to its original training mode
self.model.train(prev_model_training_mode)
return from_numpy_dataset(predictions)
def predict_single(self, batch):
predictions = defaultdict(list)
preds = self._predict(self.model, batch)
self._accumulate_preds(preds, predictions)
self._concat_preds(predictions)
return from_numpy_dataset(predictions)