def log_classification_epoch_metrics(self, metrics: MetricsDict) -> None: """ Writes all values from MetricsDict object into a file for Tensorboard visualization, and into the AzureML run context. :param metrics: dictionary containing the metrics to be logged, averaged over minibatches. """ for hue_name, label, metric in metrics.enumerate_single_values(): self.log_to_azure_and_tensorboard(get_metric_name_with_hue_prefix(label, hue_name), metric)
def test_delete_hue() -> None: h1 = "a" h2 = "b" a = MetricsDict(hues=[h1, h2]) a.add_metric("foo", 1.0, hue=h1) a.add_metric("bar", 2.0, hue=h2) a.delete_hue(h1) assert a.get_hue_names(include_default=False) == [h2] assert list(a.enumerate_single_values()) == [(h2, "bar", 2.0)]
def test_metrics_store_mixed_hues() -> None: """ Test to make sure metrics dict is able to handle default and non-default hues """ m = MetricsDict(hues=["A", "B"]) m.add_metric("foo", 1) m.add_metric("foo", 1, hue="B") m.add_metric("bar", 2, hue="A") assert list(m.enumerate_single_values()) == \ [('A', 'bar', 2), ('B', 'foo', 1), (MetricsDict.DEFAULT_HUE_KEY, 'foo', 1)]
def store_epoch_metrics( azure_and_tensorboard_logger: AzureAndTensorboardLogger, df_logger: DataframeLogger, epoch: int, metrics: MetricsDict, learning_rates: List[float], config: ModelConfigBase) -> None: """ Writes the loss, Dice scores, and learning rates into a file for Tensorboard visualization, and into the AzureML run context. :param azure_and_tensorboard_logger: An instance of AzureAndTensorboardLogger. :param df_logger: An instance of DataframeLogger, for logging results to csv. :param epoch: The epoch corresponding to the results. :param metrics: The metrics of the specified epoch, averaged along its batches. :param learning_rates: The logged learning rates. :param config: one of SegmentationModelBase """ if config.is_segmentation_model: azure_and_tensorboard_logger.log_segmentation_epoch_metrics( metrics, learning_rates) logger_row = { LoggingColumns.Dice.value: metrics.get_single_metric(MetricType.DICE), LoggingColumns.Loss.value: metrics.get_single_metric(MetricType.LOSS), LoggingColumns.SecondsPerEpoch.value: metrics.get_single_metric(MetricType.SECONDS_PER_EPOCH) } elif config.is_scalar_model: assert isinstance(metrics, MetricsDict) azure_and_tensorboard_logger.log_classification_epoch_metrics(metrics) logger_row: Dict[str, float] = {} # type: ignore for hue_name, metric_name, metric_value in metrics.enumerate_single_values( ): logging_column_name = get_column_name_for_logging( metric_name, hue_name=hue_name) logger_row[logging_column_name] = metric_value else: raise ValueError( "Model must be either classification, regression or segmentation model" ) logger_row.update({ LoggingColumns.Epoch.value: epoch, LoggingColumns.CrossValidationSplitIndex.value: config.cross_validation_split_index }) df_logger.add_record(logger_row)
def test_metrics_dict_flatten(hues: Optional[List[str]]) -> None: m = MetricsDict(hues=hues) _hues = hues or [MetricsDict.DEFAULT_HUE_KEY] * 2 m.add_metric("foo", 1.0, hue=_hues[0]) m.add_metric("foo", 2.0, hue=_hues[1]) m.add_metric("bar", 3.0, hue=_hues[0]) m.add_metric("bar", 4.0, hue=_hues[1]) if hues is None: average = m.average(across_hues=True) # We should be able to flatten out all the singleton values that the `average` operation returns all_values = list(average.enumerate_single_values()) assert all_values == [(MetricsDict.DEFAULT_HUE_KEY, "foo", 1.5), (MetricsDict.DEFAULT_HUE_KEY, "bar", 3.5)] # When trying to flatten off a dictionary that has two values, this should fail: with pytest.raises(ValueError) as ex: list(m.enumerate_single_values()) assert "only hold 1 item" in str(ex) else: average = m.average(across_hues=False) all_values = list(average.enumerate_single_values()) assert all_values == [('A', 'foo', 1.0), ('A', 'bar', 3.0), ('B', 'foo', 2.0), ('B', 'bar', 4.0)]