def _create_masked_and_check_expected( _model_outputs: torch.Tensor, _labels: torch.Tensor, _subject_ids: Sequence[IntOrString], _sorted_indices: Optional[torch.Tensor] = None) -> None: _masked = get_masked_model_outputs_and_labels(_model_outputs, _labels, _subject_ids) assert _masked is not None sorted_indices = _masked.labels.sorted_indices if _sorted_indices is None else _sorted_indices if sorted_indices is not None: _labels = _labels[sorted_indices] _model_outputs = _model_outputs[sorted_indices] _subject_ids = np.array(_subject_ids)[sorted_indices].tolist() _expected_labels = _labels.transpose(dim0=0, dim1=1).flatten() _mask = ~torch.isnan(_expected_labels) _expected_labels = _expected_labels[_mask] _expected_model_outputs = _model_outputs.transpose( dim0=0, dim1=1).flatten()[_mask] _expected_subject_ids = _subject_ids assert torch.equal(_expected_model_outputs, _masked.model_outputs.data) assert torch.equal(_expected_labels, _masked.labels.data) assert _expected_subject_ids[:_masked.labels.sorted_indices. shape[0]] == _masked.subject_ids
def forward_criterion(self, model_output: Union[torch.Tensor, List[torch.Tensor]], labels: NumpyOrTorch) -> torch.Tensor: _model_output: torch.Tensor # we need to gather the model outputs before masking them for the criterion. if isinstance(model_output, list): # When using multiple GPUs, model_output is a list of tensors. Gather will concatenate them # across the first dimension, and move them to GPU0. _model_output = torch.nn.parallel.gather(model_output, target_device=0) else: _model_output = model_output # create masked sequences based on the labels masked_model_outputs_and_labels = get_masked_model_outputs_and_labels( _model_output, labels) if masked_model_outputs_and_labels is None: raise ValueError("Invalid model_output and labels found") # do not use a data parallel criterion as we have gathered the model outputs if isinstance(self.criterion, DataParallelCriterion): criterion = self.criterion.module # type: ignore else: criterion = self.criterion return criterion(masked_model_outputs_and_labels.model_outputs, masked_model_outputs_and_labels.labels)
def compute_and_log_metrics(self, logits: torch.Tensor, targets: torch.Tensor, subject_ids: List[str], is_training: bool, metrics: ModuleDict, logger: DataframeLogger, current_epoch: int, data_split: ModelExecutionMode) -> None: """ Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers. :param logits: The model output before normalization. :param targets: The expected model outputs. :param subject_ids: The subject IDs for the present minibatch. :param is_training: If True, write the metrics as training metrics, otherwise as validation metrics. :param metrics: A dictionary mapping from names of prediction targets to a list of metric computers, as returned by create_metric_computers. :param logger: An object of type DataframeLogger which can be be used for logging within this function. :param current_epoch: Current epoch number. :param data_split: ModelExecutionMode object indicating if this is the train or validation split. :return: """ per_subject_outputs: List[Tuple[str, str, torch.Tensor, torch.Tensor]] = [] for i, (prediction_target, metric_list) in enumerate(metrics.items()): # mask the model outputs and labels if required masked = get_masked_model_outputs_and_labels( logits[:, i, ...], targets[:, i, ...], subject_ids) # compute metrics on valid masked tensors only if masked is not None: _logits = masked.model_outputs.data _posteriors = self.get_post_loss_logits_normalization_function( )(_logits) # Classification metrics expect labels as integers, but they are float throughout the rest of the code labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype _labels = masked.labels.data.to(dtype=labels_dtype) _subject_ids = masked.subject_ids assert _subject_ids is not None for metric in metric_list: if isinstance( metric, ScalarMetricsBase) and metric.compute_from_logits: metric(_logits, _labels) else: metric(_posteriors, _labels) per_subject_outputs.extend( zip(_subject_ids, [prediction_target] * len(_subject_ids), _posteriors.tolist(), _labels.tolist())) # Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current # rank in distributed training, and will be aggregated after training. for subject, prediction_target, model_output, label in per_subject_outputs: logger.add_record({ LoggingColumns.Epoch.value: current_epoch, LoggingColumns.Patient.value: subject, LoggingColumns.Hue.value: prediction_target, LoggingColumns.ModelOutput.value: model_output, LoggingColumns.Label.value: label, LoggingColumns.DataSplit.value: data_split.value })
def _forward_criterion( _logits: torch.Tensor, _labels: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: loss = self.forward_criterion_with_autocast(_logits, _labels).to( torch.float32) masked_model_outputs_and_labels = get_masked_model_outputs_and_labels( _logits, _labels) assert masked_model_outputs_and_labels is not None ece = ece_criterion( masked_model_outputs_and_labels.model_outputs.data.unsqueeze( dim=0), masked_model_outputs_and_labels.labels.data.unsqueeze(dim=0)) return loss, ece
def compute_and_log_metrics(self, logits: torch.Tensor, targets: torch.Tensor, subject_ids: List[str], is_training: bool) -> None: """ Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers. :param logits: The model output before normalization. :param targets: The expected model outputs. :param subject_ids: The subject IDs for the present minibatch. :param is_training: If True, write the metrics as training metrics, otherwise as validation metrics. :return: """ metrics = self.train_metric_computers if is_training else self.val_metric_computers per_subject_outputs: List[Tuple[str, str, torch.Tensor, torch.Tensor]] = [] for i, (prediction_target, metric_list) in enumerate(metrics.items()): # mask the model outputs and labels if required masked = get_masked_model_outputs_and_labels( logits[:, i, ...], targets[:, i, ...], subject_ids) # compute metrics on valid masked tensors only if masked is not None: _logits = masked.model_outputs.data _posteriors = self.logits_to_posterior(_logits) # Classification metrics expect labels as integers, but they are float throughout the rest of the code labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype _labels = masked.labels.data.to(dtype=labels_dtype) _subject_ids = masked.subject_ids assert _subject_ids is not None for metric in metric_list: if isinstance( metric, ScalarMetricsBase) and metric.compute_from_logits: metric(_logits, _labels) else: metric(_posteriors, _labels) per_subject_outputs.extend( zip(_subject_ids, [prediction_target] * len(_subject_ids), _posteriors.tolist(), _labels.tolist())) # Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current # rank in distributed training, and will be aggregated after training. logger = self.train_subject_outputs_logger if is_training else self.val_subject_outputs_logger data_split = ModelExecutionMode.TRAIN if is_training else ModelExecutionMode.VAL for subject, prediction_target, model_output, label in per_subject_outputs: logger.add_record({ LoggingColumns.Epoch.value: self.current_epoch, LoggingColumns.Patient.value: subject, LoggingColumns.Hue.value: prediction_target, LoggingColumns.ModelOutput.value: model_output, LoggingColumns.Label.value: label, LoggingColumns.DataSplit.value: data_split.value })
def compute_scalar_metrics(metrics_dict: ScalarMetricsDict, subject_ids: Sequence[str], model_output: torch.Tensor, labels: torch.Tensor, loss_type: ScalarLoss = ScalarLoss.BinaryCrossEntropyWithLogits) -> None: """ Computes various metrics for a binary classification task from real-valued model output and a label vector, and stores them in the given `metrics_dict`. The model output is assumed to be in the range between 0 and 1, a value larger than 0.5 indicates a prediction of class 1. The label vector is expected to contain class indices 0 and 1 only. Metrics for each model output channel will be isolated, and a non-default hue for each model output channel is expected, and must exist in the provided metrics_dict. The Default hue is used for single model outputs. :param metrics_dict: An object that holds all metrics. It will be updated in-place. :param subject_ids: Subject ids for the model output and labels. :param model_output: A tensor containing model outputs. :param labels: A tensor containing class labels. :param loss_type: The type of loss that the model uses. This is required to optionally convert 2-dim model output to probabilities. """ _model_output_channels = model_output.shape[1] model_output_hues = metrics_dict.get_hue_names(include_default=len(metrics_dict.hues_without_default) == 0) if len(model_output_hues) < _model_output_channels: raise ValueError("Hues must be provided for each model output channel, found " f"{_model_output_channels} channels but only {len(model_output_hues)} hues") for i, hue in enumerate(model_output_hues): # mask the model outputs and labels if required masked_model_outputs_and_labels = get_masked_model_outputs_and_labels( model_output[:, i, ...], labels[:, i, ...], subject_ids) # compute metrics on valid masked tensors only if masked_model_outputs_and_labels is not None: _model_output, _labels, _subject_ids = \ masked_model_outputs_and_labels.model_outputs.data, \ masked_model_outputs_and_labels.labels.data, \ masked_model_outputs_and_labels.subject_ids # Convert labels to the same datatype as the model outputs, necessary when running with AMP _labels = _labels.to(dtype=_model_output.dtype) if loss_type == ScalarLoss.MeanSquaredError: metrics = { MetricType.MEAN_SQUARED_ERROR: F.mse_loss(_model_output, _labels, reduction='mean').item(), MetricType.MEAN_ABSOLUTE_ERROR: mean_absolute_error(_model_output, _labels), MetricType.EXPLAINED_VAR: r2_score(_model_output, _labels) } else: metrics = { MetricType.CROSS_ENTROPY: F.binary_cross_entropy(_model_output, _labels, reduction='mean').item(), MetricType.ACCURACY_AT_THRESHOLD_05: binary_classification_accuracy(_model_output, _labels) } for key, value in metrics.items(): if key == MetricType.EXPLAINED_VAR: # For a batch size 1, R2 score can be nan. We need to ignore nans # when average in case the last batch is of size 1. metrics_dict.add_metric(key, value, skip_nan_when_averaging=True, hue=hue) else: metrics_dict.add_metric(key, value, hue=hue) assert _subject_ids is not None metrics_dict.add_predictions(_subject_ids, _model_output.detach().cpu().numpy(), _labels.cpu().numpy(), hue=hue)