def store_epoch_metrics(metrics: DictStrFloat,
epoch: int,
file_logger: DataframeLogger) -> None:
"""
Writes all metrics (apart from ones that measure run time) into a CSV file,
with an additional columns for epoch number.
:param file_logger: An instance of DataframeLogger, for logging results to csv.
:param epoch: The epoch corresponding to the results.
:param metrics: The metrics of the specified epoch, averaged along its batches.
"""
logger_row = {}
for key, value in metrics.items():
tokens = key.split("/")
if len(tokens) == 1:
metric_name = tokens[0]
hue_suffix = ""
elif len(tokens) == 2:
metric_name = tokens[0]
hue_suffix = "/" + tokens[1]
else:
raise ValueError(f"Expected key to have format 'metric_name[/optional_suffix_for_hue]', got {key}")
if metric_name == MetricType.SECONDS_PER_BATCH.value or metric_name == MetricType.SECONDS_PER_EPOCH.value:
continue
if metric_name in INTERNAL_TO_LOGGING_COLUMN_NAMES.keys():
logger_row[INTERNAL_TO_LOGGING_COLUMN_NAMES[metric_name].value + hue_suffix] = value
else:
logger_row[metric_name + hue_suffix] = value
logger_row[LoggingColumns.Epoch.value] = epoch
file_logger.add_record(logger_row)
file_logger.flush()
def compute_and_log_metrics(self,
logits: torch.Tensor,
targets: torch.Tensor,
subject_ids: List[str],
is_training: bool,
metrics: ModuleDict,
logger: DataframeLogger,
current_epoch: int,
data_split: ModelExecutionMode) -> None:
posteriors = self.get_post_loss_logits_normalization_function()(logits)
labels = torch.argmax(targets, dim=-1)
metric = metrics[MetricsDict.DEFAULT_HUE_KEY][0]
metric(posteriors, labels)
per_subject_outputs = zip(subject_ids, posteriors.tolist(), targets.tolist())
for subject, model_output, target in per_subject_outputs:
for i in range(len(self.target_names)):
logger.add_record({
LoggingColumns.Epoch.value: current_epoch,
LoggingColumns.Patient.value: subject,
LoggingColumns.Hue.value: self.target_names[i],
LoggingColumns.ModelOutput.value: model_output[i],
LoggingColumns.Label.value: target[i],
LoggingColumns.DataSplit.value: data_split.value
})
def compute_and_log_metrics(self, logits: torch.Tensor,
targets: torch.Tensor, subject_ids: List[str],
is_training: bool, metrics: ModuleDict,
logger: DataframeLogger, current_epoch: int,
data_split: ModelExecutionMode) -> None:
"""
Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers.
:param logits: The model output before normalization.
:param targets: The expected model outputs.
:param subject_ids: The subject IDs for the present minibatch.
:param is_training: If True, write the metrics as training metrics, otherwise as validation metrics.
:param metrics: A dictionary mapping from names of prediction targets to a list of metric computers,
as returned by create_metric_computers.
:param logger: An object of type DataframeLogger which can be be used for logging within this function.
:param current_epoch: Current epoch number.
:param data_split: ModelExecutionMode object indicating if this is the train or validation split.
:return:
"""
per_subject_outputs: List[Tuple[str, str, torch.Tensor,
torch.Tensor]] = []
for i, (prediction_target, metric_list) in enumerate(metrics.items()):
# mask the model outputs and labels if required
masked = get_masked_model_outputs_and_labels(
logits[:, i, ...], targets[:, i, ...], subject_ids)
# compute metrics on valid masked tensors only
if masked is not None:
_logits = masked.model_outputs.data
_posteriors = self.get_post_loss_logits_normalization_function(
)(_logits)
# Classification metrics expect labels as integers, but they are float throughout the rest of the code
labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype
_labels = masked.labels.data.to(dtype=labels_dtype)
_subject_ids = masked.subject_ids
assert _subject_ids is not None
for metric in metric_list:
if isinstance(
metric,
ScalarMetricsBase) and metric.compute_from_logits:
metric(_logits, _labels)
else:
metric(_posteriors, _labels)
per_subject_outputs.extend(
zip(_subject_ids, [prediction_target] * len(_subject_ids),
_posteriors.tolist(), _labels.tolist()))
# Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current
# rank in distributed training, and will be aggregated after training.
for subject, prediction_target, model_output, label in per_subject_outputs:
logger.add_record({
LoggingColumns.Epoch.value: current_epoch,
LoggingColumns.Patient.value: subject,
LoggingColumns.Hue.value: prediction_target,
LoggingColumns.ModelOutput.value: model_output,
LoggingColumns.Label.value: label,
LoggingColumns.DataSplit.value: data_split.value
})
def store_epoch_metrics(metrics: DictStrFloat, epoch: int,
file_logger: DataframeLogger) -> None:
"""
Writes all metrics into a CSV file, with an additional columns for epoch number.
:param file_logger: An instance of DataframeLogger, for logging results to csv.
:param epoch: The epoch corresponding to the results.
:param metrics: The metrics of the specified epoch, averaged along its batches.
"""
logger_row = {}
for key, value in metrics.items():
if key in INTERNAL_TO_LOGGING_COLUMN_NAMES.keys():
logger_row[INTERNAL_TO_LOGGING_COLUMN_NAMES[key].value] = value
else:
logger_row[key] = value
logger_row[LoggingColumns.Epoch.value] = epoch
file_logger.add_record(logger_row)
file_logger.flush()
def test_dataframe_logger() -> None:
fixed_columns = {"cross_validation_split_index": 1}
records = [
{
"bar": math.pi,
MetricType.LEARNING_RATE.value: 1e-5
},
{
"bar": math.pi,
MetricType.LEARNING_RATE.value: 1
},
]
out_buffer = StringIO()
df = DataframeLogger(csv_path=out_buffer, fixed_columns=fixed_columns)
for r in records:
df.add_record(r)
df.flush()
assert out_buffer.getvalue().splitlines() == [
'bar,LearningRate,cross_validation_split_index',
'3.141593,1.000000e-05,1', '3.141593,1.000000e+00,1'
]