def metric_computations_using_keras_saved_model(
model_name: Text,
model_loader: types.ModelLoader,
eval_config: Optional[config.EvalConfig],
tensor_adapter_config: Optional[
tensor_adapter.TensorAdapterConfig] = None,
batch_size: Optional[int] = None) -> metric_types.MetricComputations:
"""Returns computations for computing metrics natively using keras.
Args:
model_name: Name of model.
model_loader: Loader for shared model containing keras saved model to use
for metric computations.
eval_config: Eval config.
tensor_adapter_config: Tensor adapter config which specifies how to obtain
tensors from the Arrow RecordBatch.
batch_size: Batch size to use during evaluation (testing only).
"""
model = model_loader.load()
# If metrics were only added using model.compile then use
# model.compiled_metrics and model.compiled_loss to compute the metrics,
# otherwise custom metrics added via model.add_metric were also used and we
# need to call model.evaluate.
if not model.metrics:
return []
elif (hasattr(model, 'compiled_metrics') and model.compiled_metrics
and hasattr(model, 'compiled_loss') and model.compiled_loss
and len(model.compiled_metrics.metrics) +
len(model.compiled_loss.metrics) == len(model.metrics)):
output_names = model.output_names if hasattr(model,
'output_names') else []
keys = _metric_keys(
chain(model.compiled_metrics.metrics, model.compiled_loss.metrics),
model_name, output_names)
return [
metric_types.MetricComputation(
keys=keys,
preprocessor=None,
combiner=_KerasCompiledMetricsCombiner(keys, model_name,
model_loader,
eval_config,
batch_size))
]
else:
output_names = model.output_names if hasattr(model,
'output_names') else []
keys = _metric_keys(model.metrics, model_name, output_names)
return [
metric_types.MetricComputation(
keys=keys,
# TODO(b/178158073): By using inputs instead of batched features we
# incur the cost of having to parse the inputs a second time. In
# addition, transformed features (i.e. TFT, KPL) are not supported.
preprocessor=metric_types.InputPreprocessor(),
combiner=_KerasEvaluateCombiner(keys, model_name, model_loader,
eval_config,
tensor_adapter_config,
batch_size))
]
def _weighted_example_count(
name: Text = WEIGHTED_EXAMPLE_COUNT_NAME,
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for weighted example count."""
if model_names is None:
model_names = ['']
if output_names is None:
output_names = ['']
keys = []
computations = []
for model_name in model_names:
for output_name in output_names:
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name)
keys.append(key)
# Note: This cannot be implemented based on the weight stored in
# calibration because weighted example count is used with multi-class, etc
# models that do not use calibration metrics.
computations.append(
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_WeightedExampleCountCombiner(key)))
return computations
def _weighted_labels_predictions_examples(
name: Text = _WEIGHTED_LABELS_PREDICTIONS_EXAMPLES_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for weighted labels, predictions, and examples.
Args:
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
"""
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None, # Use default
combiner=_WeightedLabelsPredictionsExamplesCombiner(
key, eval_config=eval_config))
]
def _fixed_size_sample(
sampled_key: Text,
size: int,
name: Text,
random_seed: Optional[int],
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metrics computations for FixedSizeSample metrcs."""
keys = []
for model_name in model_names or ['']:
for output_name in output_names or ['']:
for sub_key in sub_keys or [None]:
keys.append(
metric_types.MetricKey(name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted))
return [
metric_types.MetricComputation(
keys=keys,
preprocessor=metric_types.FeaturePreprocessor(
feature_keys=[sampled_key]),
combiner=_FixedSizeSampleCombineFn(
metric_keys=keys,
sampled_key=sampled_key,
size=size,
example_weighted=example_weighted,
random_seed=random_seed))
]
def _total_attributions_computations(
absolute: bool = True,
name: Text = '',
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions.
Args:
absolute: True to use absolute value when summing.
name: Metric name.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
"""
if not name:
if absolute:
name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME
else:
name = '_' + TOTAL_ATTRIBUTIONS_NAME
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=_AttributionsPreprocessor(),
combiner=_TotalAttributionsCombiner(key, absolute))
]
def _min_label_position(
name=MIN_LABEL_POSITION_NAME,
label_key: Optional[Text] = None,
eval_config: Optional[config.EvalConfig] = None,
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None,
query_key: Text = '') -> metric_types.MetricComputations:
"""Returns metric computations for min label position."""
if not query_key:
raise ValueError(
'a query_key is required to use MinLabelPosition metric')
if model_names is None:
model_names = ['']
if output_names is None:
output_names = ['']
keys = []
computations = []
preprocessor = None
if label_key:
preprocessor = metric_types.FeaturePreprocessor(
feature_keys=[label_key])
for model_name in model_names:
for output_name in output_names:
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name)
keys.append(key)
computations.append(
metric_types.MetricComputation(
keys=[key],
preprocessor=preprocessor,
combiner=_MinLabelPositionCombiner(key, eval_config,
label_key)))
return computations
def example_count(
name: str = EXAMPLE_COUNT_NAME,
model_names: Optional[List[str]] = None,
output_names: Optional[List[str]] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for example count."""
computations = []
for model_name in model_names or ['']:
for output_name in output_names or ['']:
keys = []
for sub_key in sub_keys or [None]:
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
keys.append(key)
# Note: This cannot be implemented based on the weight stored in
# calibration because weighted example count is used with multi-class, etc
# models that do not use calibration metrics.
computations.append(
metric_types.MetricComputation(keys=keys,
preprocessor=None,
combiner=_ExampleCountCombiner(
model_name, output_name,
keys, example_weighted)))
return computations
def _total_attributions_computations(
absolute: bool = True,
name: str = '',
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions.
Args:
absolute: True to use absolute value when summing.
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
example_weighted: True if example weights should be applied.
"""
if not name:
if absolute:
name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME
else:
name = '_' + TOTAL_ATTRIBUTIONS_NAME
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=metric_types.AttributionPreprocessor(feature_keys={}),
combiner=_TotalAttributionsCombiner(key, eval_config, absolute))
]
def _multi_label_confusion_matrix_plot(
thresholds: Optional[List[float]] = None,
num_thresholds: Optional[int] = None,
name: Text = MULTI_LABEL_CONFUSION_MATRIX_PLOT_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
) -> metric_types.MetricComputations:
"""Returns computations for multi-label confusion matrix at thresholds."""
if num_thresholds is not None and thresholds is not None:
raise ValueError(
'only one of thresholds or num_thresholds can be set at a time')
if num_thresholds is None and thresholds is None:
thresholds = [0.5]
if num_thresholds is not None:
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [-_EPSILON] + thresholds + [1.0 + _EPSILON]
key = metric_types.PlotKey(name=name,
model_name=model_name,
output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_MultiLabelConfusionMatrixPlotCombiner(
key=key, eval_config=eval_config, thresholds=thresholds))
]
def _class_weights_from_labels(
class_ids: List[int],
name: Text = _CLASS_WEIGHTS_FROM_LABELS_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '') -> metric_types.MetricComputations:
"""Returns metric computations for class weights based on labels.
Args:
class_ids: List of class Ids to compute weighted labels from.
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
"""
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None, # Use default
combiner=_ClassWeightsFromLabelsCombiner(key,
eval_config=eval_config,
class_ids=class_ids))
]
def _weighted_labels_predictions_examples(
name: Text = _WEIGHTED_LABELS_PREDICTIONS_EXAMPLES_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None,
class_weights: Optional[Dict[int, float]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for weighted labels, predictions, and examples.
Args:
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
class_weights: Optional class weights to apply to multi-class / multi-label
labels and predictions prior to flattening (when micro averaging is used).
"""
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None, # Use default
combiner=_WeightedLabelsPredictionsExamplesCombiner(
key, eval_config=eval_config, class_weights=class_weights))
]
def _query_statistics(total_queries_name=TOTAL_QUERIES_NAME,
total_documents_name=TOTAL_DOCUMENTS_NAME,
min_documents_name=MIN_DOCUMENTS_NAME,
max_documents_name=MAX_DOCUMENTS_NAME,
query_key: Text = '') -> metric_types.MetricComputations:
"""Returns metric computations for query statistics."""
if not query_key:
raise ValueError(
'a query_key is required to use QueryStatistics metrics')
total_queries_key = metric_types.MetricKey(name=total_queries_name)
total_documents_key = metric_types.MetricKey(name=total_documents_name)
min_documents_key = metric_types.MetricKey(name=min_documents_name)
max_documents_key = metric_types.MetricKey(name=max_documents_name)
return [
metric_types.MetricComputation(keys=[
total_queries_key, total_documents_key, min_documents_key,
max_documents_key
],
preprocessor=None,
combiner=_QueryStatisticsCombiner(
total_queries_key,
total_documents_key,
min_documents_key,
max_documents_key))
]
def metric_computations_using_eval_saved_model(
model_name: Text,
model_loader: types.ModelLoader,
batch_size: Optional[int] = None) -> metric_types.MetricComputations:
"""Returns computations for computing metrics natively using EvalMetricsGraph.
Note that unlike other computations, there is no direct key associated with
this computation. Instead the final computation returns the actual internal
metric keys used by the model such as 'auc', etc).
Args:
model_name: Name of model.
model_loader: Loader for shared model containing eval saved model to use for
metric computations.
batch_size: Batch size to use during evaluation (testing only).
"""
return [
# _EvalSavedModelPreprocessor loads the EvalSavedModel into memory under a
# shared handle that can be used by subsequent steps. Combiner lifting and
# producer-consumer fusion should ensure that the processor and combiner
# run in the same process and memory space.
#
# TODO(b/69566045): Remove model loading from _EvalSavedModelPreprocessor
# and move model loading to _EvalSavedModelCombiner.setup after it is
# available in Beam.
metric_types.MetricComputation(
keys=[],
preprocessor=_EvalSavedModelPreprocessor(model_name, model_loader),
combiner=_EvalSavedModelCombiner(model_name, model_loader,
batch_size))
]
def calibration_histogram(
num_buckets: Optional[int] = None,
left: Optional[float] = None,
right: Optional[float] = None,
name: Text = None,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None,
aggregation_type: Optional[metric_types.AggregationType] = None,
class_weights: Optional[Dict[int, float]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for calibration histogram.
Args:
num_buckets: Number of buckets to use. Note that the actual number of
buckets will be num_buckets + 2 to account for the edge cases.
left: Start of predictions interval.
right: End of predictions interval.
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
aggregation_type: Optional aggregation type.
class_weights: Optional class weights to apply to multi-class / multi-label
labels and predictions prior to flattening (when micro averaging is used).
Returns:
MetricComputations for computing the histogram(s).
"""
if num_buckets is None:
num_buckets = DEFAULT_NUM_BUCKETS
if left is None:
left = 0.0
if right is None:
right = 1.0
if name is None:
name = '{}_{}'.format(CALIBRATION_HISTOGRAM_NAME, num_buckets)
key = metric_types.PlotKey(
name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_CalibrationHistogramCombiner(
key=key,
eval_config=eval_config,
aggregation_type=aggregation_type,
class_weights=class_weights,
num_buckets=num_buckets,
left=left,
right=right))
]
def _example_count(
name: Text = EXAMPLE_COUNT_NAME) -> metric_types.MetricComputations:
"""Returns metric computations for computing example counts."""
key = metric_types.MetricKey(name=name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=_ExampleCountPreprocessor(),
combiner=_ExampleCountCombiner(key))
]
def multi_class_confusion_matrices(
thresholds: Optional[List[float]] = None,
num_thresholds: Optional[int] = None,
name: str = MULTI_CLASS_CONFUSION_MATRICES,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns computations for multi-class confusion matrices.
Args:
thresholds: A specific set of thresholds to use. The caller is responsible
for marking the bondaires with +/-epsilon if desired. Only one of
num_thresholds or thresholds should be used.
num_thresholds: Number of thresholds to use. Thresholds will be calculated
using linear interpolation between 0.0 and 1.0 with equidistant values and
bondardaries at -epsilon and 1.0+epsilon. Values must be > 0. Only one of
num_thresholds or thresholds should be used.
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
example_weighted: True if example weights should be applied.
Raises:
ValueError: If both num_thresholds and thresholds are set at the same time.
"""
if num_thresholds is not None and thresholds is not None:
raise ValueError(
'only one of thresholds or num_thresholds can be set at a time')
if num_thresholds is None and thresholds is None:
thresholds = [0.0]
if num_thresholds is not None:
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [-_EPSILON] + thresholds + [1.0 + _EPSILON]
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_MultiClassConfusionMatrixCombiner(
key=key,
eval_config=eval_config,
example_weighted=example_weighted,
thresholds=thresholds))
]
def _query_statistics(
total_queries_name: str = TOTAL_QUERIES_NAME,
total_documents_name: str = TOTAL_DOCUMENTS_NAME,
min_documents_name: str = MIN_DOCUMENTS_NAME,
max_documents_name: str = MAX_DOCUMENTS_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
query_key: str = '',
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for query statistics."""
if not query_key:
raise ValueError(
'a query_key is required to use QueryStatistics metrics')
total_queries_key = metric_types.MetricKey(
name=total_queries_name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
total_documents_key = metric_types.MetricKey(
name=total_documents_name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
min_documents_key = metric_types.MetricKey(
name=min_documents_name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
max_documents_key = metric_types.MetricKey(
name=max_documents_name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(keys=[
total_queries_key, total_documents_key, min_documents_key,
max_documents_key
],
preprocessor=None,
combiner=_QueryStatisticsCombiner(
total_queries_key,
total_documents_key,
min_documents_key,
max_documents_key, eval_config,
model_name, output_name,
example_weighted))
]
def _binary_confusion_matrix_computation(
thresholds: List[float],
name: Optional[str] = None,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
extract_label_prediction_and_weight: Optional[Callable[
..., Any]] = metric_util.to_label_prediction_example_weight,
preprocessor: Optional[Callable[..., Any]] = None,
example_id_key: Optional[str] = None,
example_ids_count: Optional[int] = None,
aggregation_type: Optional[metric_types.AggregationType] = None,
class_weights: Optional[Dict[int, float]] = None,
example_weighted: bool = False,
fractional_labels: float = True) -> metric_types.MetricComputations:
"""Returns metric computations for computing binary confusion matrix."""
if example_ids_count is None:
example_ids_count = DEFAULT_NUM_EXAMPLE_IDS
# To generate unique name for each computation
if name is None:
name = (f'{_BINARY_CONFUSION_MATRIX_NAME}_{list(thresholds)}_'
f'{example_id_key}_{example_ids_count}')
key = metric_types.MetricKey(
name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=preprocessor,
combiner=_BinaryConfusionMatrixCombiner(
key=key,
eval_config=eval_config,
thresholds=thresholds,
extract_label_prediction_and_weight=extract_label_prediction_and_weight,
example_id_key=example_id_key,
example_ids_count=example_ids_count,
aggregation_type=aggregation_type,
class_weights=class_weights,
example_weighted=example_weighted,
fractional_labels=fractional_labels))
]
def _ndcg(gain_key: str,
top_k_list: Optional[List[int]] = None,
name: str = NDCG_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_names: Optional[List[str]] = None,
output_names: Optional[List[str]] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None,
example_weighted: bool = False,
query_key: str = '') -> metric_types.MetricComputations:
"""Returns metric computations for NDCG."""
if not query_key:
raise ValueError('a query_key is required to use NDCG metric')
sub_keys = [k for k in sub_keys if k is not None]
if top_k_list:
if sub_keys is None:
sub_keys = []
for k in top_k_list:
if not any([sub_key.top_k == k for sub_key in sub_keys]):
sub_keys.append(metric_types.SubKey(top_k=k))
if not sub_keys or any([sub_key.top_k is None for sub_key in sub_keys]):
raise ValueError(
'top_k values are required to use NDCG metric: {}'.format(sub_keys))
computations = []
for model_name in model_names if model_names else ['']:
for output_name in output_names if output_names else ['']:
keys = []
for sub_key in sub_keys:
keys.append(
metric_types.MetricKey(
name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted))
computations.append(
metric_types.MetricComputation(
keys=keys,
preprocessor=metric_types.FeaturePreprocessor(
feature_keys=[query_key, gain_key]),
combiner=_NDCGCombiner(
metric_keys=keys,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted,
query_key=query_key,
gain_key=gain_key)))
return computations
def _tjur_discrimination(
name=_TJUR_DISCRIMINATION_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
class_weights: Optional[Dict[int, float]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for TJUR discrimination."""
key = metric_types.MetricKey(
name=name, model_name=model_name, output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_TJURDiscriminationCombiner(key, eval_config, class_weights))
]
def _multi_class_confusion_matrix_at_thresholds(
thresholds: Optional[List[float]] = None,
name: Text = MULTI_CLASS_CONFUSION_MATRIX_AT_THRESHOLDS_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
) -> metric_types.MetricComputations:
"""Returns computations for multi-class confusion matrix at thresholds."""
key = metric_types.PlotKey(
name=name, model_name=model_name, output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_MultiClassConfusionMatrixAtThresholdsCombiner(
key=key, eval_config=eval_config, thresholds=thresholds))
]
def _squared_pearson_correlation(
name: Text = SQUARED_PEARSON_CORRELATION_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
class_weights: Optional[Dict[int, float]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for squared pearson correlation (r^2)."""
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_SquaredPearsonCorrelationCombiner(
key, eval_config, class_weights))
]
def _multi_label_confusion_matrix_plot(
thresholds: Optional[List[float]] = None,
name: Text = MULTI_LABEL_CONFUSION_MATRIX_PLOT_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
) -> metric_types.MetricComputations:
"""Returns computations for multi-label confusion matrix at thresholds."""
key = metric_types.PlotKey(name=name,
model_name=model_name,
output_name=output_name)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_MultiLabelConfusionMatrixPlotCombiner(
key=key, eval_config=eval_config, thresholds=thresholds))
]
def symmetric_prediction_difference_computations(
name: str = SYMMETRIC_PREDICITON_DIFFERENCE_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_names: Optional[List[str]] = None,
output_names: Optional[List[str]] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for SymmetricPredictionDifference.
This is not meant to be used with merge_per_key_computations because we
don't want to create computations for the baseline model, and we want to
provide the baseline model name to each Combiner
Args:
name: The name of the metric returned by the computations.
eval_config: The EvalConfig for this TFMA evaluation.
model_names: The set of models for which to compute this metric.
output_names: The set of output names for which to compute this metric.
sub_keys: The set of sub_key settings for which to compute this metric.
example_weighted: Whether to compute this metric using example weights.
"""
computations = []
baseline_spec = model_util.get_baseline_model_spec(eval_config)
baseline_model_name = baseline_spec.name if baseline_spec else None
for model_name in model_names or ['']:
if model_name == baseline_model_name:
continue
for output_name in output_names or ['']:
for sub_key in sub_keys or [None]:
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted,
is_diff=True)
computations.append(
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_SymmetricPredictionDifferenceCombiner(
eval_config, baseline_model_name, model_name,
output_name, key, example_weighted)))
return computations
def _exact_match(
name: Text,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None,
aggregation_type: Optional[metric_types.AggregationType] = None,
class_weights: Optional[Dict[int, float]] = None,
convert_to: Optional[Text] = None) -> metric_types.MetricComputations:
"""Returns metric computations for computing the exact match score."""
key = metric_types.MetricKey(
name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_ExactMatchCombiner(key, eval_config, aggregation_type,
class_weights, convert_to))
]
def _tjur_discrimination(
name: str = _TJUR_DISCRIMINATION_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
aggregation_type: Optional[metric_types.AggregationType] = None,
class_weights: Optional[Dict[int, float]] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for TJUR discrimination."""
key = metric_types.MetricKey(
name=name,
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_TJURDiscriminationCombiner(key, eval_config,
aggregation_type, class_weights,
example_weighted))
]
def _example_count(
name: Text = EXAMPLE_COUNT_NAME,
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for computing example counts."""
keys = []
for model_name in model_names or ['']:
for output_name in output_names or ['']:
for sub_key in sub_keys or [None]:
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
keys.append(key)
return [
metric_types.MetricComputation(
keys=keys,
preprocessor=_ExampleCountPreprocessor(),
combiner=_ExampleCountCombiner(keys))
]
def _squared_pearson_correlation(
name: str = SQUARED_PEARSON_CORRELATION_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
aggregation_type: Optional[metric_types.AggregationType] = None,
class_weights: Optional[Dict[int, float]] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for squared pearson correlation (r^2)."""
key = metric_types.MetricKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=None,
combiner=_SquaredPearsonCorrelationCombiner(
key, eval_config, aggregation_type, class_weights,
example_weighted))
]
def metric_computations_using_eval_saved_model(
model_name: Text,
model_loader: types.ModelLoader,
batch_size: Optional[int] = None) -> metric_types.MetricComputations:
"""Returns computations for computing metrics natively using EvalMetricsGraph.
Note that unlike other computations, there is no direct key associated with
this computation. Instead the final computation returns the actual internal
metric keys used by the model such as 'auc', etc).
Args:
model_name: Name of model.
model_loader: Loader for shared model containing eval saved model to use for
metric computations.
batch_size: Batch size to use during evaluation (testing only).
"""
return [
metric_types.MetricComputation(
keys=[],
preprocessor=metric_types.InputPreprocessor(),
combiner=_EvalSavedModelCombiner(model_name, model_loader,
batch_size))
]
def _ndcg(gain_key: Text,
name: Text = NDCG_NAME,
eval_config: Optional[config.EvalConfig] = None,
model_names: List[Text] = None,
output_names: List[Text] = None,
sub_keys: Optional[List[metric_types.SubKey]] = None,
query_key: Text = '') -> metric_types.MetricComputations:
"""Returns metric computations for NDCG."""
if not query_key:
raise ValueError('a query_key is required to use NDCG metric')
if sub_keys is None or any([sub_key.top_k is None for sub_key in sub_keys]):
raise ValueError(
'top_k values are required to use NDCG metric: {}'.format(sub_keys))
computations = []
for model_name in model_names if model_names else ['']:
for output_name in output_names if output_names else ['']:
keys = []
for sub_key in sub_keys:
keys.append(
metric_types.MetricKey(
name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key))
computations.append(
metric_types.MetricComputation(
keys=keys,
preprocessor=metric_types.FeaturePreprocessor(
feature_keys=[query_key, gain_key]),
combiner=_NDCGCombiner(
metric_keys=keys,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
query_key=query_key,
gain_key=gain_key)))
return computations
