def _create_masked_and_check_expected(
_model_outputs: torch.Tensor,
_labels: torch.Tensor,
_subject_ids: Sequence[IntOrString],
_sorted_indices: Optional[torch.Tensor] = None) -> None:
_masked = get_masked_model_outputs_and_labels(_model_outputs, _labels,
_subject_ids)
assert _masked is not None
sorted_indices = _masked.labels.sorted_indices if _sorted_indices is None else _sorted_indices
if sorted_indices is not None:
_labels = _labels[sorted_indices]
_model_outputs = _model_outputs[sorted_indices]
_subject_ids = np.array(_subject_ids)[sorted_indices].tolist()
_expected_labels = _labels.transpose(dim0=0, dim1=1).flatten()
_mask = ~torch.isnan(_expected_labels)
_expected_labels = _expected_labels[_mask]
_expected_model_outputs = _model_outputs.transpose(
dim0=0, dim1=1).flatten()[_mask]
_expected_subject_ids = _subject_ids
assert torch.equal(_expected_model_outputs, _masked.model_outputs.data)
assert torch.equal(_expected_labels, _masked.labels.data)
assert _expected_subject_ids[:_masked.labels.sorted_indices.
shape[0]] == _masked.subject_ids
def forward_criterion(self, model_output: Union[torch.Tensor,
List[torch.Tensor]],
labels: NumpyOrTorch) -> torch.Tensor:
_model_output: torch.Tensor
# we need to gather the model outputs before masking them for the criterion.
if isinstance(model_output, list):
# When using multiple GPUs, model_output is a list of tensors. Gather will concatenate them
# across the first dimension, and move them to GPU0.
_model_output = torch.nn.parallel.gather(model_output,
target_device=0)
else:
_model_output = model_output
# create masked sequences based on the labels
masked_model_outputs_and_labels = get_masked_model_outputs_and_labels(
_model_output, labels)
if masked_model_outputs_and_labels is None:
raise ValueError("Invalid model_output and labels found")
# do not use a data parallel criterion as we have gathered the model outputs
if isinstance(self.criterion, DataParallelCriterion):
criterion = self.criterion.module # type: ignore
else:
criterion = self.criterion
return criterion(masked_model_outputs_and_labels.model_outputs,
masked_model_outputs_and_labels.labels)
def compute_and_log_metrics(self, logits: torch.Tensor,
targets: torch.Tensor, subject_ids: List[str],
is_training: bool, metrics: ModuleDict,
logger: DataframeLogger, current_epoch: int,
data_split: ModelExecutionMode) -> None:
"""
Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers.
:param logits: The model output before normalization.
:param targets: The expected model outputs.
:param subject_ids: The subject IDs for the present minibatch.
:param is_training: If True, write the metrics as training metrics, otherwise as validation metrics.
:param metrics: A dictionary mapping from names of prediction targets to a list of metric computers,
as returned by create_metric_computers.
:param logger: An object of type DataframeLogger which can be be used for logging within this function.
:param current_epoch: Current epoch number.
:param data_split: ModelExecutionMode object indicating if this is the train or validation split.
:return:
"""
per_subject_outputs: List[Tuple[str, str, torch.Tensor,
torch.Tensor]] = []
for i, (prediction_target, metric_list) in enumerate(metrics.items()):
# mask the model outputs and labels if required
masked = get_masked_model_outputs_and_labels(
logits[:, i, ...], targets[:, i, ...], subject_ids)
# compute metrics on valid masked tensors only
if masked is not None:
_logits = masked.model_outputs.data
_posteriors = self.get_post_loss_logits_normalization_function(
)(_logits)
# Classification metrics expect labels as integers, but they are float throughout the rest of the code
labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype
_labels = masked.labels.data.to(dtype=labels_dtype)
_subject_ids = masked.subject_ids
assert _subject_ids is not None
for metric in metric_list:
if isinstance(
metric,
ScalarMetricsBase) and metric.compute_from_logits:
metric(_logits, _labels)
else:
metric(_posteriors, _labels)
per_subject_outputs.extend(
zip(_subject_ids, [prediction_target] * len(_subject_ids),
_posteriors.tolist(), _labels.tolist()))
# Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current
# rank in distributed training, and will be aggregated after training.
for subject, prediction_target, model_output, label in per_subject_outputs:
logger.add_record({
LoggingColumns.Epoch.value: current_epoch,
LoggingColumns.Patient.value: subject,
LoggingColumns.Hue.value: prediction_target,
LoggingColumns.ModelOutput.value: model_output,
LoggingColumns.Label.value: label,
LoggingColumns.DataSplit.value: data_split.value
})
def _forward_criterion(
_logits: torch.Tensor,
_labels: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
loss = self.forward_criterion_with_autocast(_logits, _labels).to(
torch.float32)
masked_model_outputs_and_labels = get_masked_model_outputs_and_labels(
_logits, _labels)
assert masked_model_outputs_and_labels is not None
ece = ece_criterion(
masked_model_outputs_and_labels.model_outputs.data.unsqueeze(
dim=0),
masked_model_outputs_and_labels.labels.data.unsqueeze(dim=0))
return loss, ece
def compute_and_log_metrics(self, logits: torch.Tensor,
targets: torch.Tensor, subject_ids: List[str],
is_training: bool) -> None:
"""
Computes all the metrics for a given (logits, labels) pair, and writes them to the loggers.
:param logits: The model output before normalization.
:param targets: The expected model outputs.
:param subject_ids: The subject IDs for the present minibatch.
:param is_training: If True, write the metrics as training metrics, otherwise as validation metrics.
:return:
"""
metrics = self.train_metric_computers if is_training else self.val_metric_computers
per_subject_outputs: List[Tuple[str, str, torch.Tensor,
torch.Tensor]] = []
for i, (prediction_target, metric_list) in enumerate(metrics.items()):
# mask the model outputs and labels if required
masked = get_masked_model_outputs_and_labels(
logits[:, i, ...], targets[:, i, ...], subject_ids)
# compute metrics on valid masked tensors only
if masked is not None:
_logits = masked.model_outputs.data
_posteriors = self.logits_to_posterior(_logits)
# Classification metrics expect labels as integers, but they are float throughout the rest of the code
labels_dtype = torch.int if self.is_classification_model else _posteriors.dtype
_labels = masked.labels.data.to(dtype=labels_dtype)
_subject_ids = masked.subject_ids
assert _subject_ids is not None
for metric in metric_list:
if isinstance(
metric,
ScalarMetricsBase) and metric.compute_from_logits:
metric(_logits, _labels)
else:
metric(_posteriors, _labels)
per_subject_outputs.extend(
zip(_subject_ids, [prediction_target] * len(_subject_ids),
_posteriors.tolist(), _labels.tolist()))
# Write a full breakdown of per-subject predictions and labels to a file. These files are local to the current
# rank in distributed training, and will be aggregated after training.
logger = self.train_subject_outputs_logger if is_training else self.val_subject_outputs_logger
data_split = ModelExecutionMode.TRAIN if is_training else ModelExecutionMode.VAL
for subject, prediction_target, model_output, label in per_subject_outputs:
logger.add_record({
LoggingColumns.Epoch.value: self.current_epoch,
LoggingColumns.Patient.value: subject,
LoggingColumns.Hue.value: prediction_target,
LoggingColumns.ModelOutput.value: model_output,
LoggingColumns.Label.value: label,
LoggingColumns.DataSplit.value: data_split.value
})
def compute_scalar_metrics(metrics_dict: ScalarMetricsDict,
subject_ids: Sequence[str],
model_output: torch.Tensor,
labels: torch.Tensor,
loss_type: ScalarLoss = ScalarLoss.BinaryCrossEntropyWithLogits) -> None:
"""
Computes various metrics for a binary classification task from real-valued model output and a label vector,
and stores them in the given `metrics_dict`.
The model output is assumed to be in the range between 0 and 1, a value larger than 0.5 indicates a prediction
of class 1. The label vector is expected to contain class indices 0 and 1 only.
Metrics for each model output channel will be isolated, and a non-default hue for each model output channel is
expected, and must exist in the provided metrics_dict. The Default hue is used for single model outputs.
:param metrics_dict: An object that holds all metrics. It will be updated in-place.
:param subject_ids: Subject ids for the model output and labels.
:param model_output: A tensor containing model outputs.
:param labels: A tensor containing class labels.
:param loss_type: The type of loss that the model uses. This is required to optionally convert 2-dim model output
to probabilities.
"""
_model_output_channels = model_output.shape[1]
model_output_hues = metrics_dict.get_hue_names(include_default=len(metrics_dict.hues_without_default) == 0)
if len(model_output_hues) < _model_output_channels:
raise ValueError("Hues must be provided for each model output channel, found "
f"{_model_output_channels} channels but only {len(model_output_hues)} hues")
for i, hue in enumerate(model_output_hues):
# mask the model outputs and labels if required
masked_model_outputs_and_labels = get_masked_model_outputs_and_labels(
model_output[:, i, ...], labels[:, i, ...], subject_ids)
# compute metrics on valid masked tensors only
if masked_model_outputs_and_labels is not None:
_model_output, _labels, _subject_ids = \
masked_model_outputs_and_labels.model_outputs.data, \
masked_model_outputs_and_labels.labels.data, \
masked_model_outputs_and_labels.subject_ids
# Convert labels to the same datatype as the model outputs, necessary when running with AMP
_labels = _labels.to(dtype=_model_output.dtype)
if loss_type == ScalarLoss.MeanSquaredError:
metrics = {
MetricType.MEAN_SQUARED_ERROR: F.mse_loss(_model_output, _labels, reduction='mean').item(),
MetricType.MEAN_ABSOLUTE_ERROR: mean_absolute_error(_model_output, _labels),
MetricType.EXPLAINED_VAR: r2_score(_model_output, _labels)
}
else:
metrics = {
MetricType.CROSS_ENTROPY: F.binary_cross_entropy(_model_output, _labels, reduction='mean').item(),
MetricType.ACCURACY_AT_THRESHOLD_05: binary_classification_accuracy(_model_output, _labels)
}
for key, value in metrics.items():
if key == MetricType.EXPLAINED_VAR:
# For a batch size 1, R2 score can be nan. We need to ignore nans
# when average in case the last batch is of size 1.
metrics_dict.add_metric(key, value, skip_nan_when_averaging=True, hue=hue)
else:
metrics_dict.add_metric(key, value, hue=hue)
assert _subject_ids is not None
metrics_dict.add_predictions(_subject_ids, _model_output.detach().cpu().numpy(),
_labels.cpu().numpy(), hue=hue)