def _init_metrics(self, metric_list):
metrics = {}
self.required_params = set(["dataset_name", "dataset_type"])
for metric in metric_list:
params = {}
if isinstance(metric, collections.abc.Mapping):
if not hasattr(metric, "type"):
raise ValueError(
"Metric {} needs to have 'type' attribute".format(
metric))
metric = metric.type
params = getattr(metric, "params", {})
else:
if not isinstance(metric, str):
raise TypeError("Metric {} has inappropriate type"
"'dict' or 'str' allowed".format(metric))
metric_cls = registry.get_metric_class(metric)
if metric_cls is None:
raise ValueError(
"No metric named {} registered to registry".format(metric))
metrics[metric] = metric_cls(**params)
self.required_params.update(metrics[metric].required_params)
return metrics
def _init_metrics(self, metric_list):
metrics = {}
self.required_params = {"dataset_name", "dataset_type"}
for metric in metric_list:
params = {}
dataset_names = []
if isinstance(metric, collections.abc.Mapping):
if "type" not in metric:
raise ValueError(
f"Metric {metric} needs to have 'type' attribute "
+ "or should be a string"
)
metric_type = key = metric.type
params = getattr(metric, "params", {})
# Support cases where uses need to give custom metric name
if "key" in metric:
key = metric.key
# One key should only be used once
if key in metrics:
raise RuntimeError(
f"Metric with type/key '{metric_type}' has been defined more "
+ "than once in metric list."
)
# a custom list of dataset where this metric will be applied
if "datasets" in metric:
dataset_names = metric.datasets
else:
if not isinstance(metric, str):
raise TypeError(
"Metric {} has inappropriate type"
"'dict' or 'str' allowed".format(metric)
)
metric_type = key = metric
metric_cls = registry.get_metric_class(metric_type)
if metric_cls is None:
raise ValueError(
f"No metric named {metric_type} registered to registry"
)
metric_instance = metric_cls(**params)
metric_instance.name = key
metric_instance.set_applicable_datasets(dataset_names)
metrics[key] = metric_instance
self.required_params.update(metrics[key].required_params)
return metrics