def evaluation(
self,
dataset: "Dataset", # noqa: F821
dataset_name: str,
metrics_log: Dict[str, Dict[str, List[TrainerMetric]]],
tables: Dict[str, List[List[str]]],
batch_size: int,
progress_tracker: ProgressTracker,
):
predictor = Predictor(self.model,
batch_size=batch_size,
horovod=self.horovod,
report_tqdm_to_ray=self.report_tqdm_to_ray)
metrics, predictions = predictor.batch_evaluation(
dataset, collect_predictions=False, dataset_name=dataset_name)
self.append_metrics(dataset_name, metrics, metrics_log, tables,
progress_tracker)
return metrics_log, tables
def evaluate(
self,
dataset=None,
data_format=None,
batch_size=128,
skip_save_unprocessed_output=True,
skip_save_predictions=True,
skip_save_eval_stats=True,
collect_predictions=False,
collect_overall_stats=False,
output_directory='results',
return_type=pd.DataFrame,
debug=False,
**kwargs
):
self._check_initialization()
logger.debug('Preprocessing')
# preprocessing
dataset, training_set_metadata = preprocess_for_prediction(
self.model_definition,
dataset=dataset,
data_format=data_format,
training_set_metadata=self.training_set_metadata,
include_outputs=True,
)
logger.debug('Predicting')
predictor = Predictor(
batch_size=batch_size, horovod=self._horovod, debug=debug
)
stats, predictions = predictor.batch_evaluation(
self.model,
dataset,
collect_predictions=collect_predictions or collect_overall_stats,
)
# calculate the overall metrics
if collect_overall_stats:
overall_stats = calculate_overall_stats(
self.model.output_features,
predictions,
dataset,
training_set_metadata
)
stats = {of_name: {**stats[of_name], **overall_stats[of_name]}
# account for presence of 'combined' key
if of_name in overall_stats else {**stats[of_name]}
for of_name in stats}
if is_on_master():
# if we are skipping all saving,
# there is no need to create a directory that will remain empty
should_create_exp_dir = not (
skip_save_unprocessed_output and
skip_save_predictions and
skip_save_eval_stats
)
if should_create_exp_dir:
os.makedirs(output_directory, exist_ok=True)
if collect_predictions:
logger.debug('Postprocessing')
postproc_predictions = postprocess(
predictions,
self.model.output_features,
self.training_set_metadata,
output_directory=output_directory,
skip_save_unprocessed_output=skip_save_unprocessed_output
or not is_on_master(),
)
else:
postproc_predictions = predictions # = {}
if is_on_master():
if postproc_predictions is not None and not skip_save_predictions:
save_prediction_outputs(postproc_predictions,
output_directory)
print_evaluation_stats(stats)
if not skip_save_eval_stats:
save_evaluation_stats(stats, output_directory)
if not skip_save_predictions or not skip_save_eval_stats:
logger.info('Saved to: {0}'.format(output_directory))
if collect_predictions:
postproc_predictions = convert_predictions(
postproc_predictions,
self.model.output_features,
self.training_set_metadata,
return_type=return_type)
return stats, postproc_predictions, output_directory
def evaluate(self,
dataset=None,
data_format=None,
batch_size=128,
skip_save_unprocessed_output=True,
skip_save_predictions=True,
skip_save_eval_stats=True,
collect_predictions=False,
collect_overall_stats=False,
output_directory='results',
return_type=pd.DataFrame,
debug=False,
**kwargs):
self._check_initialization()
logger.debug('Preprocessing')
# Added [:] to next line, before I was just assigning,
# this way I'm copying the list. If you don't do it, you are actually
# modifying the input feature list when you add output features,
# which you definitely don't want to do
features_to_load = self.model_definition['input_features'] + \
self.model_definition['output_features']
# preprocessing
# todo refactoring: maybe replace the self.model_definition paramter
# here with features_to_load
dataset, training_set_metadata = preprocess_for_prediction(
self.model_definition,
dataset=dataset,
data_format=data_format,
training_set_metadata=self.training_set_metadata,
include_outputs=True,
)
logger.debug('Predicting')
predictor = Predictor(batch_size=batch_size,
horovod=self._horovod,
debug=debug)
stats, predictions = predictor.batch_evaluation(
self.model,
dataset,
collect_predictions=collect_predictions or collect_overall_stats,
)
# calculate the overall metrics
if collect_overall_stats:
overall_stats = calculate_overall_stats(self.model.output_features,
predictions, dataset,
training_set_metadata)
stats = {
of_name: {
**stats[of_name],
**overall_stats[of_name]
}
# account for presence of 'combined' key
if of_name in overall_stats else {
**stats[of_name]
}
for of_name in stats
}
if is_on_master():
# if we are skipping all saving,
# there is no need to create a directory that will remain empty
should_create_exp_dir = not (skip_save_unprocessed_output
and skip_save_predictions
and skip_save_eval_stats)
if should_create_exp_dir:
os.makedirs(output_directory, exist_ok=True)
if collect_predictions:
logger.debug('Postprocessing')
postproc_predictions = postprocess(
predictions,
self.model.output_features,
self.training_set_metadata,
output_directory=output_directory,
skip_save_unprocessed_output=skip_save_unprocessed_output
or not is_on_master(),
)
else:
postproc_predictions = predictions # = {}
if is_on_master():
if postproc_predictions is not None and not skip_save_predictions:
save_prediction_outputs(postproc_predictions, output_directory)
print_evaluation_stats(stats)
if not skip_save_eval_stats:
save_evaluation_stats(stats, output_directory)
if not skip_save_predictions or not skip_save_eval_stats:
logger.info('Saved to: {0}'.format(output_directory))
if collect_predictions:
postproc_predictions = convert_predictions(
postproc_predictions,
self.model.output_features,
self.training_set_metadata,
return_type=return_type)
return stats, postproc_predictions, output_directory