def custom_eval_shared_model(eval_saved_model_path, model_name, eval_config, **kwargs) -> tfma.EvalSharedModel: """Returns a single custom EvalSharedModel.""" model_path = os.path.join(eval_saved_model_path, 'model.pkl') return tfma.default_eval_shared_model( eval_saved_model_path=model_path, model_name=model_name, eval_config=eval_config, custom_model_loader=types.ModelLoader( construct_fn=_custom_model_loader_fn(model_path)), add_metrics_callbacks=kwargs.get('add_metrics_callbacks'))
def createTestEvalSharedModel( self, eval_saved_model_path: Optional[Text] = None, add_metrics_callbacks: Optional[List[ types.AddMetricsCallbackType]] = None, include_default_metrics: Optional[bool] = True, example_weight_key: Optional[Union[Text, Dict[Text, Text]]] = None, additional_fetches: Optional[List[Text]] = None, tags: Optional[Text] = None, model_type: Optional[Text] = None, model_name: Text = '', rubber_stamp: Optional[bool] = False, is_baseline: Optional[bool] = False) -> types.EvalSharedModel: if not model_type: model_type = model_util.get_model_type(None, eval_saved_model_path, tags) if not tags: if model_type in (constants.TF_GENERIC, constants.TF_ESTIMATOR): model_type = constants.TF_ESTIMATOR tags = [eval_constants.EVAL_TAG] else: tags = [tf.saved_model.SERVING] return types.EvalSharedModel( model_name=model_name, model_type=model_type, model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, example_weight_key=example_weight_key, rubber_stamp=rubber_stamp, is_baseline=is_baseline, model_loader=types.ModelLoader( tags=tags, construct_fn=model_util.model_construct_fn( eval_saved_model_path=eval_saved_model_path, model_type=model_type, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, additional_fetches=additional_fetches, tags=tags)))
def createTestEvalSharedModel( self, eval_saved_model_path: Optional[Text] = None, add_metrics_callbacks: Optional[List[ types.AddMetricsCallbackType]] = None, include_default_metrics: Optional[bool] = True, example_weight_key: Optional[Union[Text, Dict[Text, Text]]] = None, additional_fetches: Optional[List[Text]] = None, tags: Optional[Text] = None) -> types.EvalSharedModel: return types.EvalSharedModel( eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, example_weight_key=example_weight_key, model_loader=types.ModelLoader( tags=tags, construct_fn=model_util.model_construct_fn( eval_saved_model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, additional_fetches=additional_fetches, tags=tags)))
def default_eval_shared_model( eval_saved_model_path: Text, add_metrics_callbacks: Optional[List[types.AddMetricsCallbackType]] = None, include_default_metrics: Optional[bool] = True, example_weight_key: Optional[Union[Text, Dict[Text, Text]]] = None, additional_fetches: Optional[List[Text]] = None, blacklist_feature_fetches: Optional[List[Text]] = None, tags: Optional[List[Text]] = None, eval_config: Optional[config.EvalConfig] = None) -> types.EvalSharedModel: """Returns default EvalSharedModel. Args: eval_saved_model_path: Path to EvalSavedModel. add_metrics_callbacks: Optional list of callbacks for adding additional metrics to the graph (see EvalSharedModel for more information on how to configure additional metrics). Metrics for example count and example weights will be added automatically. include_default_metrics: True to include the default metrics that are part of the saved model graph during evaluation. Note that eval_config.options.include_default_metrics must also be true. example_weight_key: Example weight key (single-output model) or dict of example weight keys (multi-output model) keyed by output name. additional_fetches: Prefixes of additional tensors stored in signature_def.inputs that should be fetched at prediction time. The "features" and "labels" tensors are handled automatically and should not be included. blacklist_feature_fetches: List of tensor names in the features dictionary which should be excluded from the fetches request. This is useful in scenarios where features are large (e.g. images) and can lead to excessive memory use if stored. tags: Model tags (e.g. 'serve' for serving or 'eval' for EvalSavedModel). eval_config: Eval config. Only used for setting default tags. """ if tags is None: if eval_config: # Default to serving unless all the signature_names are eval. We do not # support running with a mixture of eval and non-eval tags. signatures = [s.signature_name for s in eval_config.model_specs] if eval_constants.EVAL_TAG in signatures: if not all(s == eval_constants.EVAL_TAG for s in signatures): tf.compat.v1.logging.warning( 'mixture of eval and non-eval signatures used: ' 'eval_config={}'.format(eval_config)) tags = [eval_constants.EVAL_TAG] else: tags = [tf.saved_model.SERVING] else: tags = [eval_constants.EVAL_TAG] # Backwards compatibility for legacy add_metrics_callbacks implementation. if tags == [eval_constants.EVAL_TAG]: # PyType doesn't know about the magic exports we do in post_export_metrics. # Additionally, the lines seem to get reordered in compilation, so we can't # just put the disable-attr on the add_metrics_callbacks lines. # pytype: disable=module-attr if not add_metrics_callbacks: add_metrics_callbacks = [] # Always compute example weight and example count. example_count_callback = post_export_metrics.example_count() add_metrics_callbacks.append(example_count_callback) if example_weight_key: if isinstance(example_weight_key, dict): for output_name, key in example_weight_key.items(): example_weight_callback = post_export_metrics.example_weight( key, metric_tag=output_name) add_metrics_callbacks.append(example_weight_callback) else: example_weight_callback = post_export_metrics.example_weight( example_weight_key) add_metrics_callbacks.append(example_weight_callback) # pytype: enable=module-attr return types.EvalSharedModel( model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, example_weight_key=example_weight_key, additional_fetches=additional_fetches, model_loader=types.ModelLoader( tags=tags, construct_fn=model_util.model_construct_fn( eval_saved_model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, additional_fetches=additional_fetches, blacklist_feature_fetches=blacklist_feature_fetches, tags=tags)))
def testWithDefaultMetricsProvidedByModel(self): export_dir = os.path.join(self._getTempDir(), 'export_dir') dummy_layer = tf.keras.layers.Input(shape=(1, )) model = tf.keras.models.Model([dummy_layer], [dummy_layer]) model.compile(loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.MeanSquaredError(name='mse')]) model.save(export_dir, save_format='tf') model_loader = types.ModelLoader( tags=[tf.saved_model.SERVING], construct_fn=model_util.model_construct_fn( eval_saved_model_path=export_dir, tags=[tf.saved_model.SERVING])) computations = tf_metric_wrapper.tf_metric_computations( [tf.keras.metrics.AUC(name='auc')], config.EvalConfig(), model_loader=model_loader) confusion_histogram = computations[0] confusion_matrix = computations[1].result confusion_metrics = computations[2].result non_confusion_metrics = computations[3] example1 = { 'labels': np.array([0.0]), 'predictions': np.array([0.0]), 'example_weights': np.array([1.0]), } example2 = { 'labels': np.array([0.0]), 'predictions': np.array([0.5]), 'example_weights': np.array([1.0]), } example3 = { 'labels': np.array([1.0]), 'predictions': np.array([0.3]), 'example_weights': np.array([1.0]), } example4 = { 'labels': np.array([1.0]), 'predictions': np.array([0.9]), 'example_weights': np.array([1.0]), } with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter sliced_examples = ( pipeline | 'Create' >> beam.Create( [example1, example2, example3, example4]) | 'Process' >> beam.Map(metric_util.to_standard_metric_inputs) | 'AddSlice' >> beam.Map(lambda x: ((), x))) confusion_result = ( sliced_examples | 'ComputeHistogram' >> beam.CombinePerKey( confusion_histogram.combiner) | 'ComputeConfusionMatrix' >> beam.Map(lambda x: (x[0], confusion_matrix(x[1]))) # pyformat: disable | 'ComputeMetric' >> beam.Map(lambda x: (x[0], confusion_metrics(x[1]))) ) # pyformat: disable non_confusion_result = (sliced_examples | 'Combine' >> beam.CombinePerKey( non_confusion_metrics.combiner)) # pylint: enable=no-value-for-parameter def check_confusion_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) auc_key = metric_types.MetricKey(name='auc') self.assertDictElementsAlmostEqual(got_metrics, {auc_key: 0.75}, places=5) except AssertionError as err: raise util.BeamAssertException(err) def check_non_confusion_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) mse_key = metric_types.MetricKey(name='mse') binary_crossentropy_key = metric_types.MetricKey( name='binary_crossentropy') self.assertDictElementsAlmostEqual( got_metrics, { mse_key: 0.1875, binary_crossentropy_key: 0.0 }, places=5) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(confusion_result, check_confusion_result, label='confusion') util.assert_that(non_confusion_result, check_non_confusion_result, label='non_confusion')
def default_eval_shared_model( eval_saved_model_path: Text, add_metrics_callbacks: Optional[List[types.AddMetricsCallbackType]] = None, include_default_metrics: Optional[bool] = True, example_weight_key: Optional[Union[Text, Dict[Text, Text]]] = None, additional_fetches: Optional[List[Text]] = None, blacklist_feature_fetches: Optional[List[Text]] = None ) -> types.EvalSharedModel: """Returns default EvalSharedModel. Args: eval_saved_model_path: Path to EvalSavedModel. add_metrics_callbacks: Optional list of callbacks for adding additional metrics to the graph (see EvalSharedModel for more information on how to configure additional metrics). Metrics for example count and example weights will be added automatically. include_default_metrics: True to include the default metrics that are part of the saved model graph during evaluation. example_weight_key: Example weight key (single-output model) or dict of example weight keys (multi-output model) keyed by output name. additional_fetches: Prefixes of additional tensors stored in signature_def.inputs that should be fetched at prediction time. The "features" and "labels" tensors are handled automatically and should not be included. blacklist_feature_fetches: List of tensor names in the features dictionary which should be excluded from the fetches request. This is useful in scenarios where features are large (e.g. images) and can lead to excessive memory use if stored. """ # Always compute example weight and example count. # PyType doesn't know about the magic exports we do in post_export_metrics. # Additionally, the lines seem to get reordered in compilation, so we can't # just put the disable-attr on the add_metrics_callbacks lines. # pytype: disable=module-attr if not add_metrics_callbacks: add_metrics_callbacks = [] example_count_callback = post_export_metrics.example_count() add_metrics_callbacks.append(example_count_callback) if example_weight_key: if isinstance(example_weight_key, dict): for output_name, key in example_weight_key.items(): example_weight_callback = post_export_metrics.example_weight( key, metric_tag=output_name) add_metrics_callbacks.append(example_weight_callback) else: example_weight_callback = post_export_metrics.example_weight( example_weight_key) add_metrics_callbacks.append(example_weight_callback) # pytype: enable=module-attr return types.EvalSharedModel( model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, example_weight_key=example_weight_key, additional_fetches=additional_fetches, model_loader=types.ModelLoader( construct_fn=model_util.model_construct_fn( eval_saved_model_path=eval_saved_model_path, add_metrics_callbacks=add_metrics_callbacks, include_default_metrics=include_default_metrics, additional_fetches=additional_fetches, blacklist_feature_fetches=blacklist_feature_fetches)))