def _yield_classification_specs(self, dataset): yield from super()._yield_classification_specs(dataset) yield base.MeasurementSpec( dataset, "average_pairwise_diversity(normalize_disagreement=True)") yield base.MeasurementSpec( dataset, "average_pairwise_diversity(normalize_disagreement=False)")
def required_measurements(self): rescaling_methods = ["temperature_scaling"] for rescaling_method in rescaling_methods: yield base.MeasurementSpec("imagenet_a", rescaling_method) yield base.MeasurementSpec("imagenet_r", rescaling_method) yield base.MeasurementSpec( "imagenet_v2(variant='MATCHED_FREQUENCY')", rescaling_method)
def required_measurements(self): rescaling_methods = ["temperature_scaling"] for rescaling_method in rescaling_methods: metric = f"{rescaling_method}" yield base.MeasurementSpec("imagenet(split='validation[:20%]')", metric) yield base.MeasurementSpec("imagenet(split='validation[20%:]')", metric)
def required_measurements(self): rescaling_methods = ["temperature_scaling"] severities = list(range(1, 6)) for rescaling_method in rescaling_methods: metric = f"{rescaling_method}" for corruption_type in IMAGENET_C_CORRUPTIONS: for severity in severities: dataset = ( f"imagenet_c(corruption_type={corruption_type!r}," f"severity={severity}," "split='validation[:20%]')") dataset = ( f"imagenet_c(corruption_type={corruption_type!r}," f"severity={severity}," "split='validation[20%:]')") yield base.MeasurementSpec(dataset, metric)
def _yield_classification_specs(dataset): for metric in [ "accuracy", "ece", "nll", "brier", "timing", "adaptive_ece(datapoints_per_bin=100,threshold=0.0)" ]: yield base.MeasurementSpec(dataset, metric)
def required_measurements(self): metrics = ["auc_pr", "auc_roc", "fpr95"] for dataset in self._datasets: for metric in metrics: yield base.MeasurementSpec(dataset, metric)
def required_measurements(self): for variant in ["rotation", "size", "location"]: dataset = f"synthetic(variant={variant!r})" yield base.MeasurementSpec(dataset, "synthetic")
def _yield_classification_specs(self, dataset): for metric in ["accuracy", "ece", "nll", "brier", "timing"]: yield base.MeasurementSpec(dataset, metric)
def _yield_classification_specs(self, dataset, use_dataset_labelset=None): """Yields a MeasurementSpec for each metric and a given dataset.""" for metric in self._yield_metrics_to_evaluate( use_dataset_labelset=use_dataset_labelset): yield base.MeasurementSpec(dataset, metric)
def required_measurements(self): yield from super().required_measurements yield base.MeasurementSpec("objectnet", "objectnet_accuracy") for gce_spec in self._yield_gce_metrics(): yield base.MeasurementSpec("objectnet", f"objectnet_{gce_spec}")