def log_classification_epoch_metrics(self, metrics: MetricsDict) -> None:
"""
Writes all values from MetricsDict object into a file for Tensorboard visualization,
and into the AzureML run context.
:param metrics: dictionary containing the metrics to be logged, averaged over minibatches.
"""
for hue_name, label, metric in metrics.enumerate_single_values():
self.log_to_azure_and_tensorboard(get_metric_name_with_hue_prefix(label, hue_name), metric)
def test_delete_hue() -> None:
h1 = "a"
h2 = "b"
a = MetricsDict(hues=[h1, h2])
a.add_metric("foo", 1.0, hue=h1)
a.add_metric("bar", 2.0, hue=h2)
a.delete_hue(h1)
assert a.get_hue_names(include_default=False) == [h2]
assert list(a.enumerate_single_values()) == [(h2, "bar", 2.0)]
def test_metrics_store_mixed_hues() -> None:
"""
Test to make sure metrics dict is able to handle default and non-default hues
"""
m = MetricsDict(hues=["A", "B"])
m.add_metric("foo", 1)
m.add_metric("foo", 1, hue="B")
m.add_metric("bar", 2, hue="A")
assert list(m.enumerate_single_values()) == \
[('A', 'bar', 2), ('B', 'foo', 1), (MetricsDict.DEFAULT_HUE_KEY, 'foo', 1)]
def store_epoch_metrics(
azure_and_tensorboard_logger: AzureAndTensorboardLogger,
df_logger: DataframeLogger, epoch: int, metrics: MetricsDict,
learning_rates: List[float], config: ModelConfigBase) -> None:
"""
Writes the loss, Dice scores, and learning rates into a file for Tensorboard visualization,
and into the AzureML run context.
:param azure_and_tensorboard_logger: An instance of AzureAndTensorboardLogger.
:param df_logger: An instance of DataframeLogger, for logging results to csv.
:param epoch: The epoch corresponding to the results.
:param metrics: The metrics of the specified epoch, averaged along its batches.
:param learning_rates: The logged learning rates.
:param config: one of SegmentationModelBase
"""
if config.is_segmentation_model:
azure_and_tensorboard_logger.log_segmentation_epoch_metrics(
metrics, learning_rates)
logger_row = {
LoggingColumns.Dice.value:
metrics.get_single_metric(MetricType.DICE),
LoggingColumns.Loss.value:
metrics.get_single_metric(MetricType.LOSS),
LoggingColumns.SecondsPerEpoch.value:
metrics.get_single_metric(MetricType.SECONDS_PER_EPOCH)
}
elif config.is_scalar_model:
assert isinstance(metrics, MetricsDict)
azure_and_tensorboard_logger.log_classification_epoch_metrics(metrics)
logger_row: Dict[str, float] = {} # type: ignore
for hue_name, metric_name, metric_value in metrics.enumerate_single_values(
):
logging_column_name = get_column_name_for_logging(
metric_name, hue_name=hue_name)
logger_row[logging_column_name] = metric_value
else:
raise ValueError(
"Model must be either classification, regression or segmentation model"
)
logger_row.update({
LoggingColumns.Epoch.value:
epoch,
LoggingColumns.CrossValidationSplitIndex.value:
config.cross_validation_split_index
})
df_logger.add_record(logger_row)
def test_metrics_dict_flatten(hues: Optional[List[str]]) -> None:
m = MetricsDict(hues=hues)
_hues = hues or [MetricsDict.DEFAULT_HUE_KEY] * 2
m.add_metric("foo", 1.0, hue=_hues[0])
m.add_metric("foo", 2.0, hue=_hues[1])
m.add_metric("bar", 3.0, hue=_hues[0])
m.add_metric("bar", 4.0, hue=_hues[1])
if hues is None:
average = m.average(across_hues=True)
# We should be able to flatten out all the singleton values that the `average` operation returns
all_values = list(average.enumerate_single_values())
assert all_values == [(MetricsDict.DEFAULT_HUE_KEY, "foo", 1.5), (MetricsDict.DEFAULT_HUE_KEY, "bar", 3.5)]
# When trying to flatten off a dictionary that has two values, this should fail:
with pytest.raises(ValueError) as ex:
list(m.enumerate_single_values())
assert "only hold 1 item" in str(ex)
else:
average = m.average(across_hues=False)
all_values = list(average.enumerate_single_values())
assert all_values == [('A', 'foo', 1.0), ('A', 'bar', 3.0), ('B', 'foo', 2.0), ('B', 'bar', 4.0)]