def metric_instance(
metric_config: config.MetricConfig,
tfma_metric_classes: Optional[Dict[Text, Type[metric_types.Metric]]] = None
) -> metric_types.Metric:
"""Creates instance of metric associated with config."""
if tfma_metric_classes is None:
tfma_metric_classes = metric_types.registered_metrics()
if metric_config.class_name in tfma_metric_classes:
return _deserialize_tfma_metric(metric_config, tfma_metric_classes)
elif not metric_config.module:
return _deserialize_tf_metric(metric_config, {})
else:
cls = getattr(importlib.import_module(metric_config.module),
metric_config.class_name)
if issubclass(cls, tf.keras.metrics.Metric):
return _deserialize_tf_metric(metric_config,
{metric_config.class_name: cls})
elif issubclass(cls, tf.keras.losses.Loss):
return _deserialize_tf_loss(metric_config,
{metric_config.class_name: cls})
elif issubclass(cls, metric_types.Metric):
return _deserialize_tfma_metric(metric_config,
{metric_config.class_name: cls})
else:
raise NotImplementedError(
'unknown metric type {}: metric={}'.format(cls, metric_config))
def has_attributions_metrics(
metrics_specs: Iterable[config_pb2.MetricsSpec]) -> bool:
"""Returns true if any of the metrics_specs have attributions metrics."""
tfma_metric_classes = metric_types.registered_metrics()
for metrics_spec in metrics_specs:
for metric_config in metrics_spec.metrics:
instance = metric_specs.metric_instance(metric_config,
tfma_metric_classes)
if isinstance(instance, AttributionsMetric):
return True
return False
def _keys_and_metrics_from_specs(
metrics_specs: Iterable[config.MetricsSpec]
) -> Iterator[Tuple[metric_types.MetricKey, config.MetricConfig,
metric_types.Metric]]:
"""Yields key, config, instance tuples for each non-diff metric in specs."""
tfma_metric_classes = metric_types.registered_metrics()
for spec in metrics_specs:
for aggregation_type, sub_keys in _create_sub_keys(spec).items():
for metric_config in spec.metrics:
instance = metric_instance(metric_config, tfma_metric_classes)
for key in _keys_for_metric(instance.name, spec, aggregation_type,
sub_keys):
yield key, metric_config, instance
def _keys_and_metrics_from_specs(
metrics_specs: Iterable[config.MetricsSpec]
) -> Iterator[Tuple[metric_types.MetricKey, config.MetricConfig,
metric_types.Metric]]:
"""Yields key, config, instance tuples for each non-diff metric in specs."""
tfma_metric_classes = metric_types.registered_metrics()
for spec in metrics_specs:
sub_keys = _create_sub_keys(spec) or [None]
if spec.aggregate.macro_average or spec.aggregate.weighted_macro_average:
sub_keys.append(None)
for metric_config in spec.metrics:
if metric_config.class_name in tfma_metric_classes:
instance = _deserialize_tfma_metric(metric_config,
tfma_metric_classes)
elif not metric_config.module:
instance = _deserialize_tf_metric(metric_config, {})
else:
cls = getattr(importlib.import_module(metric_config.module),
metric_config.class_name)
if issubclass(cls, tf.keras.metrics.Metric):
instance = _deserialize_tf_metric(
metric_config, {metric_config.class_name: cls})
elif issubclass(cls, tf.keras.losses.Loss):
instance = _deserialize_tf_loss(
metric_config, {metric_config.class_name: cls})
elif issubclass(cls, metric_types.Metric):
instance = _deserialize_tfma_metric(
metric_config, {metric_config.class_name: cls})
else:
raise NotImplementedError(
'unknown metric type {}: metric={}'.format(
cls, metric_config))
if (hasattr(instance, 'is_model_independent')
and instance.is_model_independent()):
key = metric_types.MetricKey(name=instance.name)
yield key, metric_config, instance
else:
for key in _keys_for_metric(instance.name, spec, sub_keys):
yield key, metric_config, instance
def to_computations(
metrics_specs: List[config.MetricsSpec],
eval_config: Optional[config.EvalConfig] = None,
schema: Optional[schema_pb2.Schema] = None
) -> metric_types.MetricComputations:
"""Returns computations associated with given metrics specs."""
computations = []
#
# Split into TF metrics and TFMA metrics
#
# Dict[Text, Type[tf.keras.metrics.Metric]]
tf_metric_classes = {} # class_name -> class
# Dict[Text, Type[tf.keras.losses.Loss]]
tf_loss_classes = {} # class_name -> class
# List[metric_types.MetricsSpec]
tf_metrics_specs = []
# Dict[Text, Type[metric_types.Metric]]
tfma_metric_classes = metric_types.registered_metrics() # class_name -> class
# List[metric_types.MetricsSpec]
tfma_metrics_specs = []
#
# Note: Lists are used instead of Dicts for the following items because
# protos are are no hashable.
#
# List[List[_TFOrTFMAMetric]] (offsets align with metrics_specs).
per_spec_metric_instances = []
# List[List[_TFMetricOrLoss]] (offsets align with tf_metrics_specs).
per_tf_spec_metric_instances = []
# List[List[metric_types.Metric]]] (offsets align with tfma_metrics_specs).
per_tfma_spec_metric_instances = []
for spec in metrics_specs:
tf_spec = config.MetricsSpec()
tf_spec.CopyFrom(spec)
del tf_spec.metrics[:]
tfma_spec = config.MetricsSpec()
tfma_spec.CopyFrom(spec)
del tfma_spec.metrics[:]
for metric in spec.metrics:
if metric.class_name in tfma_metric_classes:
tfma_spec.metrics.append(metric)
elif not metric.module:
tf_spec.metrics.append(metric)
else:
cls = getattr(importlib.import_module(metric.module), metric.class_name)
if issubclass(cls, tf.keras.metrics.Metric):
tf_metric_classes[metric.class_name] = cls
tf_spec.metrics.append(metric)
elif issubclass(cls, tf.keras.losses.Loss):
tf_loss_classes[metric.class_name] = cls
tf_spec.metrics.append(metric)
else:
tfma_metric_classes[metric.class_name] = cls
tfma_spec.metrics.append(metric)
metric_instances = []
if tf_spec.metrics:
tf_metrics_specs.append(tf_spec)
tf_metric_instances = []
for m in tf_spec.metrics:
# To distinguish losses from metrics, losses are required to set the
# module name.
if m.module == _TF_LOSSES_MODULE:
tf_metric_instances.append(_deserialize_tf_loss(m, tf_loss_classes))
else:
tf_metric_instances.append(
_deserialize_tf_metric(m, tf_metric_classes))
per_tf_spec_metric_instances.append(tf_metric_instances)
metric_instances.extend(tf_metric_instances)
if tfma_spec.metrics:
tfma_metrics_specs.append(tfma_spec)
tfma_metric_instances = [
_deserialize_tfma_metric(m, tfma_metric_classes)
for m in tfma_spec.metrics
]
per_tfma_spec_metric_instances.append(tfma_metric_instances)
metric_instances.extend(tfma_metric_instances)
per_spec_metric_instances.append(metric_instances)
# Process TF specs
computations.extend(
_process_tf_metrics_specs(tf_metrics_specs, per_tf_spec_metric_instances,
eval_config))
# Process TFMA specs
computations.extend(
_process_tfma_metrics_specs(tfma_metrics_specs,
per_tfma_spec_metric_instances, eval_config,
schema))
# Process aggregation based metrics (output aggregation and macro averaging).
# Note that processing of TF and TFMA specs were setup to create the binarized
# metrics that macro averaging depends on.
for i, spec in enumerate(metrics_specs):
for aggregation_type, sub_keys in _create_sub_keys(spec).items():
output_names = spec.output_names or ['']
output_weights = dict(spec.output_weights)
if not set(output_weights.keys()).issubset(output_names):
raise ValueError(
'one or more output_names used in output_weights does not exist: '
'output_names={}, output_weights={}'.format(output_names,
output_weights))
for model_name in spec.model_names or ['']:
for sub_key in sub_keys:
for metric in per_spec_metric_instances[i]:
if (aggregation_type and (aggregation_type.macro_average or
aggregation_type.weighted_macro_average)):
class_weights = _class_weights(spec) or {}
for output_name in output_names:
sub_keys = _macro_average_sub_keys(sub_key, class_weights)
if aggregation_type.macro_average:
computations.extend(
aggregation.macro_average(
metric.get_config()['name'],
sub_keys=sub_keys,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
class_weights=class_weights))
elif aggregation_type.weighted_macro_average:
computations.extend(
aggregation.weighted_macro_average(
metric.get_config()['name'],
sub_keys=sub_keys,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
class_weights=class_weights))
if output_weights:
computations.extend(
aggregation.output_average(
metric.get_config()['name'],
output_weights=output_weights,
eval_config=eval_config,
model_name=model_name,
sub_key=sub_key))
return computations
def to_computations(
metrics_specs: List[config.MetricsSpec],
eval_config: Optional[config.EvalConfig] = None,
schema: Optional[schema_pb2.Schema] = None
) -> metric_types.MetricComputations:
"""Returns computations associated with given metrics specs."""
computations = []
#
# Split into TF metrics and TFMA metrics
#
# Dict[Text, Type[tf.keras.metrics.Metric]]
tf_metric_classes = {} # class_name -> class
# Dict[Text, Type[tf.keras.losses.Loss]]
tf_loss_classes = {} # class_name -> class
# List[metric_types.MetricsSpec]
tf_metrics_specs = []
# Dict[Text, Type[metric_types.Metric]]
tfma_metric_classes = metric_types.registered_metrics(
) # class_name -> class
# List[metric_types.MetricsSpec]
tfma_metrics_specs = []
#
# Note: Lists are used instead of Dicts for the following items because
# protos are are no hashable.
#
# List[List[_TFOrTFMAMetric]] (offsets align with metrics_specs).
per_spec_metric_instances = []
# List[List[_TFMetricOrLoss]] (offsets align with tf_metrics_specs).
per_tf_spec_metric_instances = []
# List[List[metric_types.Metric]]] (offsets align with tfma_metrics_specs).
per_tfma_spec_metric_instances = []
for spec in metrics_specs:
tf_spec = config.MetricsSpec()
tf_spec.CopyFrom(spec)
del tf_spec.metrics[:]
tfma_spec = config.MetricsSpec()
tfma_spec.CopyFrom(spec)
del tfma_spec.metrics[:]
for metric in spec.metrics:
if metric.class_name in tfma_metric_classes:
tfma_spec.metrics.append(metric)
elif not metric.module:
tf_spec.metrics.append(metric)
else:
cls = getattr(importlib.import_module(metric.module),
metric.class_name)
if issubclass(cls, tf.keras.metrics.Metric):
tf_metric_classes[metric.class_name] = cls
tf_spec.metrics.append(metric)
elif issubclass(cls, tf.keras.losses.Loss):
tf_loss_classes[metric.class_name] = cls
tf_spec.metrics.append(metric)
else:
tfma_metric_classes[metric.class_name] = cls
tfma_spec.metrics.append(metric)
metric_instances = []
if tf_spec.metrics:
tf_metrics_specs.append(tf_spec)
tf_metric_instances = []
for m in tf_spec.metrics:
# To distinguish losses from metrics, losses are required to set the
# module name.
if m.module == _TF_LOSSES_MODULE:
tf_metric_instances.append(
_deserialize_tf_loss(m, tf_loss_classes))
else:
tf_metric_instances.append(
_deserialize_tf_metric(m, tf_metric_classes))
per_tf_spec_metric_instances.append(tf_metric_instances)
metric_instances.extend(tf_metric_instances)
if tfma_spec.metrics:
tfma_metrics_specs.append(tfma_spec)
tfma_metric_instances = [
_deserialize_tfma_metric(m, tfma_metric_classes)
for m in tfma_spec.metrics
]
per_tfma_spec_metric_instances.append(tfma_metric_instances)
metric_instances.extend(tfma_metric_instances)
per_spec_metric_instances.append(metric_instances)
#
# Group TF metrics by the subkeys, models and outputs. This is done in reverse
# because model and subkey processing is done outside of TF and so each unique
# sub key combination needs to be run through a separate model instance. Note
# that output_names are handled by the tf_metric_computation since all the
# outputs are batch calculated in a single model evaluation call.
#
# Dict[metric_types.SubKey, Dict[Text, List[int]]
tf_spec_indices_by_subkey = {
} # SubKey -> model_name -> [index(MetricSpec)]
for i, spec in enumerate(tf_metrics_specs):
sub_keys = _create_sub_keys(spec)
if not sub_keys:
sub_keys = [None]
for sub_key in sub_keys:
if sub_key not in tf_spec_indices_by_subkey:
tf_spec_indices_by_subkey[sub_key] = {}
# Dict[Text, List[config.MetricSpec]]
tf_spec_indices_by_model = (tf_spec_indices_by_subkey[sub_key]
) # name -> [ModelSpec]
model_names = spec.model_names
if not model_names:
model_names = [''
] # '' is name used when only one model is used
for model_name in model_names:
if model_name not in tf_spec_indices_by_model:
tf_spec_indices_by_model[model_name] = []
tf_spec_indices_by_model[model_name].append(i)
for sub_key, spec_indices_by_model in tf_spec_indices_by_subkey.items():
for model_name, indices in spec_indices_by_model.items():
# Class weights are a dict that is not hashable, so we store index to spec
# containing class weights.
metrics_by_class_weights_by_output = collections.defaultdict(dict)
for i in indices:
class_weights_i = None
if tf_metrics_specs[i].HasField('aggregate'):
class_weights_i = i
metrics_by_output = metrics_by_class_weights_by_output[
class_weights_i]
output_names = [''] # '' is name used when only one output
if tf_metrics_specs[i].output_names:
output_names = tf_metrics_specs[i].output_names
for output_name in output_names:
if output_name not in metrics_by_output:
metrics_by_output[output_name] = []
metrics_by_output[output_name].extend(
per_tf_spec_metric_instances[i])
for i, metrics_by_output in metrics_by_class_weights_by_output.items(
):
class_weights = None
if i is not None:
class_weights = dict(
tf_metrics_specs[i].aggregate.class_weights)
computations.extend(
tf_metric_wrapper.tf_metric_computations(
metrics_by_output,
eval_config=eval_config,
model_name=model_name,
sub_key=sub_key,
class_weights=class_weights))
#
# Group TFMA metric specs by the metric classes
#
# Dict[bytes, List[config.MetricSpec]]
tfma_specs_by_metric_config = {} # hash(MetricConfig) -> [MetricSpec]
# Dict[bytes, metric_types.Metric]
hashed_metrics = {} # hash(MetricConfig) -> Metric
for i, spec in enumerate(tfma_metrics_specs):
for metric_config, metric in zip(spec.metrics,
per_tfma_spec_metric_instances[i]):
# Note that hashing by SerializeToString() is only safe if used within the
# same process.
config_hash = metric_config.SerializeToString()
if config_hash not in tfma_specs_by_metric_config:
hashed_metrics[config_hash] = metric
tfma_specs_by_metric_config[config_hash] = []
tfma_specs_by_metric_config[config_hash].append(spec)
for config_hash, specs in tfma_specs_by_metric_config.items():
metric = hashed_metrics[config_hash]
for spec in specs:
sub_keys = _create_sub_keys(spec)
class_weights = None
if spec.HasField('aggregate'):
class_weights = dict(spec.aggregate.class_weights)
computations.extend(
metric.computations(
eval_config=eval_config,
schema=schema,
model_names=spec.model_names if spec.model_names else [''],
output_names=spec.output_names
if spec.output_names else [''],
sub_keys=sub_keys,
class_weights=class_weights,
query_key=spec.query_key))
#
# Create macro averaging metrics
#
for i, spec in enumerate(metrics_specs):
if spec.aggregate.macro_average or spec.aggregate.weighted_macro_average:
sub_keys = _create_sub_keys(spec)
if sub_keys is None:
raise ValueError(
'binarize settings are required when aggregate.macro_average or '
'aggregate.weighted_macro_average is used: spec={}'.format(
spec))
for model_name in spec.model_names or ['']:
for output_name in spec.output_names or ['']:
for metric in per_spec_metric_instances[i]:
if spec.aggregate.macro_average:
computations.extend(
aggregation.macro_average(
metric.get_config()['name'],
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_keys=sub_keys,
class_weights=dict(
spec.aggregate.class_weights)))
elif spec.aggregate.weighted_macro_average:
computations.extend(
aggregation.weighted_macro_average(
metric.get_config()['name'],
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_keys=sub_keys,
class_weights=dict(
spec.aggregate.class_weights)))
return computations
def to_computations(
metrics_specs: List[config.MetricsSpec],
eval_config: Optional[config.EvalConfig] = None,
model_loaders: Optional[Dict[Text, types.ModelLoader]] = None
) -> metric_types.MetricComputations:
"""Returns computations associated with given metrics specs."""
computations = []
#
# Split into TF metrics and TFMA metrics
#
# Dict[Text, Type[tf.keras.metrics.Metric]]
tf_metric_classes = {} # class_name -> class
# List[metric_types.MetricsSpec]
tf_metrics_specs = []
# Dict[Text, Type[metric_types.Metric]]
tfma_metric_classes = metric_types.registered_metrics(
) # class_name -> class
# List[metric_types.MetricsSpec]
tfma_metrics_specs = []
for spec in metrics_specs:
tf_spec = config.MetricsSpec()
tf_spec.CopyFrom(spec)
del tf_spec.metrics[:]
tfma_spec = config.MetricsSpec()
tfma_spec.CopyFrom(spec)
del tfma_spec.metrics[:]
for metric in spec.metrics:
if metric.class_name in tfma_metric_classes:
tfma_spec.metrics.append(metric)
elif not metric.module:
tf_spec.metrics.append(metric)
else:
cls = getattr(importlib.import_module(metric.module_name),
metric.class_name)
if isinstance(metric, tf.keras.metrics.Metric):
tf_metric_classes[metric.class_name] = cls
tf_spec.metrics.append(metric)
else:
tfma_metric_classes[metric.class_name] = cls
tfma_spec.metrics.append(metric)
if tf_spec.metrics:
tf_metrics_specs.append(tf_spec)
if tfma_spec.metrics:
tfma_metrics_specs.append(tfma_spec)
#
# Group TF metrics by the subkeys, models and outputs. This is done in reverse
# because model and subkey processing is done outside of TF and so each unique
# sub key combination needs to be run through a separate model instance. Note
# that output_names are handled by the tf_metric_computation since all the
# outputs are batch calculated in a single model evaluation call.
#
# Dict[metric_types.SubKey, Dict[Text, List[config.MetricSpec]]
tf_specs_by_subkey = {} # SubKey -> model_name -> [MetricSpec]
for spec in tf_metrics_specs:
sub_keys = _create_sub_keys(spec)
if not sub_keys:
sub_keys = [None]
for sub_key in sub_keys:
if sub_key not in tf_specs_by_subkey:
tf_specs_by_subkey[sub_key] = {}
# Dict[Text, List[config.MetricSpec]]
tf_specs_by_model = tf_specs_by_subkey[
sub_key] # name -> [ModelSpec]
model_names = spec.model_names
if not model_names:
model_names = [''
] # '' is name used when only one model is used
for model_name in model_names:
if model_name not in tf_specs_by_model:
tf_specs_by_model[model_name] = []
tf_specs_by_model[model_name].append(spec)
for sub_key, specs_by_model in tf_specs_by_subkey.items():
for model_name, specs in specs_by_model.items():
metrics_by_output = {}
for spec in specs:
metrics = [
_deserialize_tf_metric(m, tf_metric_classes)
for m in spec.metrics
]
if spec.output_names:
for output_name in spec.output_names:
if output_name not in metrics_by_output:
metrics_by_output[output_name] = []
metrics_by_output[output_name].extend(metrics)
else:
if '' not in metrics_by_output:
metrics_by_output[''] = [
] # '' is name used when only one output
metrics_by_output[''].extend(metrics)
model_loader = None
if model_loaders and model_name in model_loaders:
model_loader = model_loaders[model_name]
computations.extend(
tf_metric_wrapper.tf_metric_computations(
metrics_by_output,
eval_config=eval_config,
model_name=model_name,
sub_key=sub_key,
model_loader=model_loader))
#
# Group TFMA metric specs by the metric classes
#
# Dict[bytes, List[config.MetricSpec]]
tfma_specs_by_metric_config = {} # hash(MetricConfig) -> [MetricSpec]
# Dict[bytes, config.MetricConfig]
hashed_metric_configs = {} # hash(MetricConfig) -> MetricConfig
for spec in tfma_metrics_specs:
for metric_config in spec.metrics:
# Note that hashing by SerializeToString() is only safe if used within the
# same process.
config_hash = metric_config.SerializeToString()
if config_hash not in tfma_specs_by_metric_config:
hashed_metric_configs[config_hash] = metric_config
tfma_specs_by_metric_config[config_hash] = []
tfma_specs_by_metric_config[config_hash].append(spec)
for config_hash, specs in tfma_specs_by_metric_config.items():
metric = _deserialize_tfma_metric(hashed_metric_configs[config_hash],
tfma_metric_classes)
for spec in specs:
sub_keys = _create_sub_keys(spec)
computations.extend(
metric.computations(
eval_config=eval_config,
model_names=spec.model_names if spec.model_names else [''],
output_names=spec.output_names
if spec.output_names else [''],
sub_keys=sub_keys,
query_key=spec.query_key))
return computations
def metric_thresholds_from_metrics_specs(
metrics_specs: List[config.MetricsSpec]
) -> Dict[metric_types.MetricKey, Union[config.GenericChangeThreshold,
config.GenericValueThreshold]]:
"""Returns thresholds associated with given metrics specs."""
result = {}
tfma_metric_classes = metric_types.registered_metrics()
for spec in metrics_specs:
sub_keys = _create_sub_keys(spec) or [None]
if spec.aggregate.macro_average or spec.aggregate.weighted_macro_average:
sub_keys.append(None)
# Add thresholds for metrics computed in-graph.
for metric_name, threshold in spec.thresholds.items():
for model_name in spec.model_names or ['']:
for output_name in spec.output_names or ['']:
for sub_key in sub_keys:
if threshold.HasField('value_threshold'):
key = metric_types.MetricKey(
name=metric_name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
is_diff=False)
result[key] = threshold.value_threshold
if threshold.HasField('change_threshold'):
key = metric_types.MetricKey(
name=metric_name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
is_diff=True)
result[key] = threshold.change_threshold
# Thresholds in MetricConfig override thresholds in MetricsSpec.
for metric in spec.metrics:
if not metric.HasField('threshold'):
continue
if metric.class_name in tfma_metric_classes:
instance = _deserialize_tfma_metric(metric,
tfma_metric_classes)
elif not metric.module:
instance = _deserialize_tf_metric(metric, {})
else:
cls = getattr(importlib.import_module(metric.module),
metric.class_name)
if issubclass(cls, tf.keras.metrics.Metric):
instance = _deserialize_tf_metric(metric,
{metric.class_name: cls})
elif issubclass(cls, tf.keras.losses.Loss):
instance = _deserialize_tf_loss(metric,
{metric.class_name: cls})
elif isinstance(metric, metric_types.Metric):
instance = _deserialize_tfma_metric(
metric, {metric.class_name: cls})
else:
raise NotImplementedError(
'unknown metric type {}: metric={}'.format(
type(metric), metric))
if (hasattr(instance, 'is_model_independent')
and instance.is_model_independent()):
if metric.threshold.HasField('value_threshold'):
key = metric_types.MetricKey(name=instance.name,
is_diff=False)
result[key] = metric.threshold.value_threshold
else:
for model_name in spec.model_names or ['']:
for output_name in spec.output_names or ['']:
for sub_key in sub_keys:
if metric.threshold.HasField('value_threshold'):
key = metric_types.MetricKey(
name=instance.name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
is_diff=False)
result[key] = metric.threshold.value_threshold
if metric.threshold.HasField('change_threshold'):
key = metric_types.MetricKey(
name=instance.name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
is_diff=True)
result[key] = metric.threshold.change_threshold
return result
