def store_epoch_metrics(metrics: DictStrFloat, epoch: int, file_logger: DataframeLogger) -> None: """ Writes all metrics (apart from ones that measure run time) into a CSV file, with an additional columns for epoch number. :param file_logger: An instance of DataframeLogger, for logging results to csv. :param epoch: The epoch corresponding to the results. :param metrics: The metrics of the specified epoch, averaged along its batches. """ logger_row = {} for key, value in metrics.items(): tokens = key.split("/") if len(tokens) == 1: metric_name = tokens[0] hue_suffix = "" elif len(tokens) == 2: metric_name = tokens[0] hue_suffix = "/" + tokens[1] else: raise ValueError(f"Expected key to have format 'metric_name[/optional_suffix_for_hue]', got {key}") if metric_name == MetricType.SECONDS_PER_BATCH.value or metric_name == MetricType.SECONDS_PER_EPOCH.value: continue if metric_name in INTERNAL_TO_LOGGING_COLUMN_NAMES.keys(): logger_row[INTERNAL_TO_LOGGING_COLUMN_NAMES[metric_name].value + hue_suffix] = value else: logger_row[metric_name + hue_suffix] = value logger_row[LoggingColumns.Epoch.value] = epoch file_logger.add_record(logger_row) file_logger.flush()
def compute_and_log_metrics(self, logits: torch.Tensor, targets: torch.Tensor, subject_ids: List[str], is_training: bool, metrics: ModuleDict, logger: DataframeLogger, current_epoch: int, data_split: ModelExecutionMode) -> None: posteriors = self.get_post_loss_logits_normalization_function()(logits) labels = torch.argmax(targets, dim=-1) metric = metrics[MetricsDict.DEFAULT_HUE_KEY][0] metric(posteriors, labels) per_subject_outputs = zip(subject_ids, posteriors.tolist(), targets.tolist()) for subject, model_output, target in per_subject_outputs: for i in range(len(self.target_names)): logger.add_record({ LoggingColumns.Epoch.value: current_epoch, LoggingColumns.Patient.value: subject, LoggingColumns.Hue.value: self.target_names[i], LoggingColumns.ModelOutput.value: model_output[i], LoggingColumns.Label.value: target[i], LoggingColumns.DataSplit.value: data_split.value })
def compute_and_log_metrics(self, logits: torch.Tensor, targets: torch.Tensor, subject_ids: List[str], is_training: bool, metrics: ModuleDict, logger: DataframeLogger, current_epoch: int, data_split: ModelExecutionMode) -> None: """ Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers. :param logits: The model output before normalization. :param targets: The expected model outputs. :param subject_ids: The subject IDs for the present minibatch. :param is_training: If True, write the metrics as training metrics, otherwise as validation metrics. :param metrics: A dictionary mapping from names of prediction targets to a list of metric computers, as returned by create_metric_computers. :param logger: An object of type DataframeLogger which can be be used for logging within this function. :param current_epoch: Current epoch number. :param data_split: ModelExecutionMode object indicating if this is the train or validation split. :return: """ per_subject_outputs: List[Tuple[str, str, torch.Tensor, torch.Tensor]] = [] for i, (prediction_target, metric_list) in enumerate(metrics.items()): # mask the model outputs and labels if required masked = get_masked_model_outputs_and_labels( logits[:, i, ...], targets[:, i, ...], subject_ids) # compute metrics on valid masked tensors only if masked is not None: _logits = masked.model_outputs.data _posteriors = self.get_post_loss_logits_normalization_function( )(_logits) # Classification metrics expect labels as integers, but they are float throughout the rest of the code labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype _labels = masked.labels.data.to(dtype=labels_dtype) _subject_ids = masked.subject_ids assert _subject_ids is not None for metric in metric_list: if isinstance( metric, ScalarMetricsBase) and metric.compute_from_logits: metric(_logits, _labels) else: metric(_posteriors, _labels) per_subject_outputs.extend( zip(_subject_ids, [prediction_target] * len(_subject_ids), _posteriors.tolist(), _labels.tolist())) # Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current # rank in distributed training, and will be aggregated after training. for subject, prediction_target, model_output, label in per_subject_outputs: logger.add_record({ LoggingColumns.Epoch.value: current_epoch, LoggingColumns.Patient.value: subject, LoggingColumns.Hue.value: prediction_target, LoggingColumns.ModelOutput.value: model_output, LoggingColumns.Label.value: label, LoggingColumns.DataSplit.value: data_split.value })
def store_epoch_metrics(metrics: DictStrFloat, epoch: int, file_logger: DataframeLogger) -> None: """ Writes all metrics into a CSV file, with an additional columns for epoch number. :param file_logger: An instance of DataframeLogger, for logging results to csv. :param epoch: The epoch corresponding to the results. :param metrics: The metrics of the specified epoch, averaged along its batches. """ logger_row = {} for key, value in metrics.items(): if key in INTERNAL_TO_LOGGING_COLUMN_NAMES.keys(): logger_row[INTERNAL_TO_LOGGING_COLUMN_NAMES[key].value] = value else: logger_row[key] = value logger_row[LoggingColumns.Epoch.value] = epoch file_logger.add_record(logger_row) file_logger.flush()
def test_dataframe_logger() -> None: fixed_columns = {"cross_validation_split_index": 1} records = [ { "bar": math.pi, MetricType.LEARNING_RATE.value: 1e-5 }, { "bar": math.pi, MetricType.LEARNING_RATE.value: 1 }, ] out_buffer = StringIO() df = DataframeLogger(csv_path=out_buffer, fixed_columns=fixed_columns) for r in records: df.add_record(r) df.flush() assert out_buffer.getvalue().splitlines() == [ 'bar,LearningRate,cross_validation_split_index', '3.141593,1.000000e-05,1', '3.141593,1.000000e+00,1' ]