# Value of a Tensor fetched using session.run.
FetchedTensorValue = Union[tf.SparseTensorValue, np.ndarray]
# Dictionary of Tensor values fetched.
# The dictionary maps original dictionary keys => ('node' => value).
DictOfFetchedTensorValues = Dict[KeyType, Dict[Text, FetchedTensorValue]]
MetricVariablesType = List[Any]
# Used in building the model diagnostics table, a MatrializedColumn is a value
# inside of ExampleAndExtract that will be emitted to file. Note that for
# strings, the values are raw byte strings rather than unicode strings. This is
# by design, as features can have arbitrary bytes values.
MaterializedColumn = NamedTuple(
'MaterializedColumn',
[('name', Text),
('value', Union[List[bytes], List[int], List[float], bytes, int, float])])
# Used in building model diagnostics table, the ExampleAndExtracts holds an
# example and all its "extractions." Extractions that should be emitted to file.
# Each Extract has a name, stored as the key of the DictOfExtractedValues.
DictOfExtractedValues = Dict[Text, Any]
# pylint: enable=invalid-name
def is_tensor(obj):
return isinstance(obj, tf.Tensor) or isinstance(obj, tf.SparseTensor)
class EvalSharedModel(
def __new__(
cls,
value,
lower_bound=None,
upper_bound=None,
unsampled_value=None,
):
# Add bounds checking?
return super(ValueWithConfidenceInterval,
cls).__new__(cls, value, lower_bound, upper_bound,
unsampled_value)
FeaturesPredictionsLabels = NamedTuple(
'FeaturesPredictionsLabels', [('input_ref', int),
('features', DictOfFetchedTensorValues),
('predictions', DictOfFetchedTensorValues),
('labels', DictOfFetchedTensorValues)])
# Used in building the model diagnostics table, a MaterializedColumn is a value
# inside of Extracts that will be emitted to file. Note that for strings, the
# values are raw byte strings rather than unicode strings. This is by design, as
# features can have arbitrary bytes values.
MaterializedColumn = NamedTuple(
'MaterializedColumn',
[('name', Text),
('value', Union[List[bytes], List[int], List[float], bytes, int, float])])
# Extracts represent data extracted during pipeline processing. In order to
# provide a flexible API, these types are just dicts where the keys are defined
# (reserved for use) by different extractor implementations. For example, the
class EvalSharedModel(
NamedTuple(
'EvalSharedModel',
[
('model_path', Text),
('add_metrics_callbacks',
List[Callable]), # List[AnyMetricsCallbackType]
('include_default_metrics', bool),
('example_weight_key', Text),
('shared_handle', shared.Shared),
('construct_fn', Callable)
])):
# pyformat: disable
"""Shared model used during extraction and evaluation.
Attributes:
model_path: Path to EvalSavedModel (containing the saved_model.pb file).
add_metrics_callbacks: Optional list of callbacks for adding additional
metrics to the graph. The names of the metrics added by the callbacks
should not conflict with existing metrics. See below for more details
about what each callback should do. The callbacks are only used during
evaluation.
include_default_metrics: True to include the default metrics that are part
of the saved model graph during evaluation.
example_weight_key: The key of the example weight column. If None, weight
will be 1 for each example.
shared_handle: Optional handle to a shared.Shared object for sharing the
in-memory model within / between stages.
construct_fn: A callable which creates a construct function
to set up the tensorflow graph. Callable takes a beam.metrics distribution
to track graph construction time.
More details on add_metrics_callbacks:
Each add_metrics_callback should have the following prototype:
def add_metrics_callback(features_dict, predictions_dict, labels_dict):
Note that features_dict, predictions_dict and labels_dict are not
necessarily dictionaries - they might also be Tensors, depending on what the
model's eval_input_receiver_fn returns.
It should create and return a metric_ops dictionary, such that
metric_ops['metric_name'] = (value_op, update_op), just as in the Trainer.
Short example:
def add_metrics_callback(features_dict, predictions_dict, labels):
metrics_ops = {}
metric_ops['mean_label'] = tf.metrics.mean(labels)
metric_ops['mean_probability'] = tf.metrics.mean(tf.slice(
predictions_dict['probabilities'], [0, 1], [2, 1]))
return metric_ops
"""
# pyformat: enable
def __new__(cls,
model_path=None,
add_metrics_callbacks=None,
include_default_metrics=True,
example_weight_key=None,
shared_handle=None,
construct_fn=None):
if not add_metrics_callbacks:
add_metrics_callbacks = []
if not shared_handle:
shared_handle = shared.Shared()
return super(EvalSharedModel,
cls).__new__(cls, model_path, add_metrics_callbacks,
include_default_metrics, example_weight_key,
shared_handle, construct_fn)
import apache_beam as beam
from tensorflow_model_analysis import types
from tensorflow_model_analysis.types_compat import List, NamedTuple, Optional, Text
# Tag for the last extractor in list of extractors.
LAST_EXTRACTOR_STAGE_NAME = '<last-extractor>'
# An Extractor is a PTransform that takes Extracts as input and returns Extracts
# as output. A typical example is a PredictExtractor that receives an 'input'
# placeholder for input and adds additional 'features', 'labels', and
# 'predictions' extracts.
Extractor = NamedTuple( # pylint: disable=invalid-name
'Extractor',
[
('stage_name', Text),
# PTransform Extracts -> Extracts
('ptransform', beam.PTransform)
])
@beam.ptransform_fn
@beam.typehints.with_input_types(beam.typehints.Any)
@beam.typehints.with_output_types(beam.typehints.Any)
def Filter(extracts, include=None, exclude=None):
"""Filters extracts to include/exclude specified keys.
Args:
extracts: PCollection of extracts.
include: Keys to include in output.
exclude: Keys to exclude from output.
class EvalInputReceiver(
NamedTuple('EvalInputReceiver',
[('features', types.TensorTypeMaybeDict),
('receiver_tensors', types.TensorTypeMaybeDict),
('labels', types.TensorTypeMaybeDict)])):
"""A return type for eval_input_receiver_fn.
from __future__ import print_function
import numpy as np
import tensorflow as tf
from tensorflow_model_analysis import types
from tensorflow_model_analysis.eval_saved_model import constants
from tensorflow_model_analysis.eval_saved_model import encoding
from tensorflow_model_analysis.eval_saved_model import graph_ref
from tensorflow_model_analysis.types_compat import Any, Dict, List, NamedTuple, Optional, Tuple # pytype: disable=not-supported-yet
from tensorflow.core.protobuf import meta_graph_pb2
FeaturesPredictionsLabels = NamedTuple( # pylint: disable=invalid-name
'FeaturesPredictionsLabels',
[('features', types.DictOfFetchedTensorValues),
('predictions', types.DictOfFetchedTensorValues),
('labels', types.DictOfFetchedTensorValues)])
class EvalSavedModel(object):
"""Abstraction for using a EvalSavedModel."""
def __init__(self, path):
self._path = path
self._graph = tf.Graph()
self._session = tf.Session(graph=self._graph)
self._load_and_parse_graph()
def _check_version(self, meta_graph_def):
version = meta_graph_def.collection_def.get(
import abc
import itertools
import tensorflow as tf
from tensorflow_model_analysis import types
from tensorflow_model_analysis import util as general_util
from tensorflow_model_analysis.eval_saved_model import constants
from tensorflow_model_analysis.eval_saved_model import encoding
from tensorflow_model_analysis.eval_saved_model import util
from tensorflow_model_analysis.types_compat import Any, Dict, List, NamedTuple, Text, Tuple
# Config for defining the input tensor feed into the EvalMetricsGraph. This
# is needed for model agnostic use cases where a graph must be constructed.
FPLFeedConfig = NamedTuple( # pylint: disable=invalid-name
'FPLFeedConfig', [('features', Dict[Text, Any]),
('predictions', Dict[Text, Any]),
('labels', Dict[Text, Any])])
class EvalMetricsGraph(object):
"""Abstraction for a graph that is used for computing and aggregating metrics.
This abstract class contains methods and lays out the API to handle metrics
computation and aggregation as part of the TFMA flow. Inheritors of this class
are responsible for setting up the metrics graph and setting the class
variables which are required to do metric calculations.
"""
__metaclass__ = abc.ABCMeta
def __init__(self):
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import apache_beam as beam
from tensorflow_model_analysis.evaluators import evaluator
from tensorflow_model_analysis.validators import validator
from tensorflow_model_analysis.types_compat import NamedTuple, Text, Union
# A writer is a PTransform that takes Evaluation or Validation output as input
# and serializes the associated PCollections of data to a sink.
Writer = NamedTuple(
'Writer',
[
('stage_name', Text),
# PTransform Evaluation -> PDone or Validation -> PDone
('ptransform', beam.PTransform)
])
@beam.ptransform_fn
@beam.typehints.with_input_types(evaluator.Evaluation)
@beam.typehints.with_output_types(beam.pvalue.PDone)
def Write(evaluation_or_validation, key, ptransform):
"""Writes given Evaluation or Validation data using given writer PTransform.
Args:
evaluation_or_validation: Evaluation or Validation data.
key: Key for Evaluation or Validation output to write. It is valid for the
key to not exist in the dict (in which case the write is a no-op).
okay = False
if minor < 11:
okay = False
if not okay:
raise RuntimeError(
'Tensorflow version >= 1.11, < 2 is required. Found (%s). Please '
'install the latest 1.x version from '
'https://github.com/tensorflow/tensorflow. ' % tf.__version__)
EvalConfig = NamedTuple( # pylint: disable=invalid-name
'EvalConfig',
[
('model_location',
Text), # The location of the model used for this evaluation
('data_location',
Text), # The location of the data used for this evaluation
('slice_spec', Optional[List[slicer.SingleSliceSpec]]
), # The corresponding slice spec
('example_weight_metric_key',
Text), # The name of the metric that contains example weight
])
def _check_version(raw_final_dict, path):
version = raw_final_dict.get(_VERSION_KEY)
if version is None:
raise ValueError(
'could not find TFMA version in raw deserialized dictionary for '
'file at %s' % path)
# We don't actually do any checking for now, since we don't have any
# compatibility issues.
from tensorflow.core.protobuf import meta_graph_pb2
# pylint: disable=invalid-name
# Type used to feed a single input to the model. This will be converted to a
# MultipleInputFeedType using a batch of size one.
SingleInputFeedType = Union[bytes, Dict[bytes, Any]]
# Type used to feed a batch of inputs to the model. This should match the values
# expected by the receiver_tensor placeholders used with the EvalInputReceiver.
# Typically this will be a batch of serialized `tf.train.Example` protos.
MultipleInputFeedType = Union[List[bytes], Dict[bytes, List[Any]]]
# Type used to return the tensor values fetched from the model.
FetchedTensorValues = NamedTuple(
'FetchedTensorValues',
[
('input_ref', int),
# Dict is keyed by group ('features', 'predictions', 'labels', etc).
('values', Dict[Text, types.TensorValueMaybeDict])
])
# pylint: enable=invalid-name
class EvalSavedModel(eval_metrics_graph.EvalMetricsGraph):
"""Abstraction for using a EvalSavedModel.
Note that this class overrides eval_metrics_graph.EvalMetricsGraph
which contains most of the functionality for handling the metric ops.
In the eval saved model path, a common graph is shared between generation
FeaturesPredictionLabels through Predict and doing the metric ops.
The specific methods of this class constructs the graph and handles
