def _yield_classification_specs(self, dataset):
yield from super()._yield_classification_specs(dataset)
yield base.MeasurementSpec(
dataset, "average_pairwise_diversity(normalize_disagreement=True)")
yield base.MeasurementSpec(
dataset,
"average_pairwise_diversity(normalize_disagreement=False)")
def required_measurements(self):
rescaling_methods = ["temperature_scaling"]
for rescaling_method in rescaling_methods:
yield base.MeasurementSpec("imagenet_a", rescaling_method)
yield base.MeasurementSpec("imagenet_r", rescaling_method)
yield base.MeasurementSpec(
"imagenet_v2(variant='MATCHED_FREQUENCY')", rescaling_method)
def required_measurements(self):
rescaling_methods = ["temperature_scaling"]
for rescaling_method in rescaling_methods:
metric = f"{rescaling_method}"
yield base.MeasurementSpec("imagenet(split='validation[:20%]')",
metric)
yield base.MeasurementSpec("imagenet(split='validation[20%:]')",
metric)
def required_measurements(self):
rescaling_methods = ["temperature_scaling"]
severities = list(range(1, 6))
for rescaling_method in rescaling_methods:
metric = f"{rescaling_method}"
for corruption_type in IMAGENET_C_CORRUPTIONS:
for severity in severities:
dataset = (
f"imagenet_c(corruption_type={corruption_type!r},"
f"severity={severity},"
"split='validation[:20%]')")
dataset = (
f"imagenet_c(corruption_type={corruption_type!r},"
f"severity={severity},"
"split='validation[20%:]')")
yield base.MeasurementSpec(dataset, metric)
def _yield_classification_specs(dataset):
for metric in [
"accuracy", "ece", "nll", "brier", "timing",
"adaptive_ece(datapoints_per_bin=100,threshold=0.0)"
]:
yield base.MeasurementSpec(dataset, metric)
def required_measurements(self):
metrics = ["auc_pr", "auc_roc", "fpr95"]
for dataset in self._datasets:
for metric in metrics:
yield base.MeasurementSpec(dataset, metric)
def required_measurements(self):
for variant in ["rotation", "size", "location"]:
dataset = f"synthetic(variant={variant!r})"
yield base.MeasurementSpec(dataset, "synthetic")
def _yield_classification_specs(self, dataset):
for metric in ["accuracy", "ece", "nll", "brier", "timing"]:
yield base.MeasurementSpec(dataset, metric)
def _yield_classification_specs(self, dataset, use_dataset_labelset=None):
"""Yields a MeasurementSpec for each metric and a given dataset."""
for metric in self._yield_metrics_to_evaluate(
use_dataset_labelset=use_dataset_labelset):
yield base.MeasurementSpec(dataset, metric)
def required_measurements(self):
yield from super().required_measurements
yield base.MeasurementSpec("objectnet", "objectnet_accuracy")
for gce_spec in self._yield_gce_metrics():
yield base.MeasurementSpec("objectnet", f"objectnet_{gce_spec}")