def __init__(
self,
examples: types.Channel = None,
model_exports: types.Channel = None,
feature_slicing_spec: Optional[evaluator_pb2.FeatureSlicingSpec] = None,
output: Optional[types.Channel] = None,
model: Optional[types.Channel] = None,
name: Optional[Text] = None):
"""Construct an Evaluator component.
Args:
examples: A Channel of 'ExamplesPath' type, usually produced by ExampleGen
component (required).
model_exports: A Channel of 'ModelExportPath' type, usually produced by
Trainer component (required).
feature_slicing_spec: Optional evaluator_pb2.FeatureSlicingSpec instance,
providing the way to slice the data.
output: Optional channel of 'ModelEvalPath' for result of evaluation.
model: Forwards compatibility alias for the 'model_exports' argument.
name: Optional unique name. Necessary if multiple Evaluator components are
declared in the same pipeline.
"""
model_exports = model_exports or model
output = output or types.Channel(
type=standard_artifacts.ModelEvaluation,
artifacts=[standard_artifacts.ModelEvaluation()])
spec = EvaluatorSpec(
examples=examples,
model_exports=model_exports,
feature_slicing_spec=(feature_slicing_spec or
evaluator_pb2.FeatureSlicingSpec()),
output=output)
super(Evaluator, self).__init__(spec=spec, name=name)
def __init__(self,
examples: types.Channel,
model_exports: types.Channel,
feature_slicing_spec: Optional[
evaluator_pb2.FeatureSlicingSpec] = None,
output: Optional[types.Channel] = None,
name: Optional[Text] = None):
"""Construct an Evaluator component.
Args:
examples: A Channel of 'ExamplesPath' type, usually produced by ExampleGen
component.
model_exports: A Channel of 'ModelExportPath' type, usually produced by
Trainer component.
feature_slicing_spec: Optional evaluator_pb2.FeatureSlicingSpec instance,
providing the way to slice the data.
output: Optional channel of 'ModelEvalPath' for result of evaluation.
name: Optional unique name. Necessary if multiple Evaluator components are
declared in the same pipeline.
"""
output = output or types.Channel(
type=standard_artifacts.ModelEvalResult,
artifacts=[standard_artifacts.ModelEvalResult()])
spec = EvaluatorSpec(
examples=channel_utils.as_channel(examples),
model_exports=channel_utils.as_channel(model_exports),
feature_slicing_spec=(feature_slicing_spec
or evaluator_pb2.FeatureSlicingSpec()),
output=output)
super(Evaluator, self).__init__(spec=spec, name=name)
def __init__(
self,
examples: channel.Channel,
model_exports: channel.Channel,
feature_slicing_spec: Optional[evaluator_pb2.FeatureSlicingSpec] = None,
name: Optional[Text] = None,
outputs: Optional[base_component.ComponentOutputs] = None):
component_name = 'Evaluator'
input_dict = {
'examples': channel.as_channel(examples),
'model_exports': channel.as_channel(model_exports),
}
exec_properties = {
'feature_slicing_spec':
json_format.MessageToJson(feature_slicing_spec or
evaluator_pb2.FeatureSlicingSpec()),
}
super(Evaluator, self).__init__(
component_name=component_name,
unique_name=name,
driver=base_driver.BaseDriver,
executor=executor.Executor,
input_dict=input_dict,
outputs=outputs,
exec_properties=exec_properties)
def testTaxiPipelineNewStyleCompatibility(self):
examples = external_input('/tmp/fake/path')
example_gen = CsvExampleGen(input=examples)
self.assertIs(example_gen.inputs['input'],
example_gen.inputs['input_base'])
statistics_gen = StatisticsGen(
examples=example_gen.outputs['examples'])
self.assertIs(statistics_gen.inputs['examples'],
statistics_gen.inputs['input_data'])
infer_schema = SchemaGen(
statistics=statistics_gen.outputs['statistics'])
self.assertIs(infer_schema.inputs['statistics'],
infer_schema.inputs['stats'])
self.assertIs(infer_schema.outputs['schema'],
infer_schema.outputs['output'])
validate_examples = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'])
self.assertIs(validate_examples.inputs['statistics'],
validate_examples.inputs['stats'])
self.assertIs(validate_examples.outputs['anomalies'],
validate_examples.outputs['output'])
transform = Transform(examples=example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file='/tmp/fake/module/file')
self.assertIs(transform.inputs['examples'],
transform.inputs['input_data'])
self.assertIs(transform.outputs['transform_graph'],
transform.outputs['transform_output'])
trainer = Trainer(
module_file='/tmp/fake/module/file',
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
self.assertIs(trainer.inputs['transform_graph'],
trainer.inputs['transform_output'])
self.assertIs(trainer.outputs['model'], trainer.outputs['output'])
evaluator = Evaluator(
examples=example_gen.outputs['examples'],
model=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
self.assertIs(evaluator.inputs['model'],
evaluator.inputs['model_exports'])
model_validator = ModelValidator(
examples=example_gen.outputs['examples'],
model=trainer.outputs['model'])
pusher = Pusher(model=trainer.outputs['output'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory='/fake/serving/dir')))
self.assertIs(pusher.inputs['model'], pusher.inputs['model_export'])
self.assertIs(pusher.outputs['pushed_model'],
pusher.outputs['model_push'])
def Do(self, input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, Any]) -> None:
"""Runs a batch job to evaluate the eval_model against the given input.
Args:
input_dict: Input dict from input key to a list of Artifacts.
- model_exports: exported model.
- examples: examples for eval the model.
output_dict: Output dict from output key to a list of Artifacts.
- output: model evaluation results.
exec_properties: A dict of execution properties.
- feature_slicing_spec: JSON string of evaluator_pb2.FeatureSlicingSpec
instance, providing the way to slice the data.
Returns:
None
"""
if 'model_exports' not in input_dict:
raise ValueError('\'model_exports\' is missing in input dict.')
if 'examples' not in input_dict:
raise ValueError('\'examples\' is missing in input dict.')
if 'output' not in output_dict:
raise ValueError('\'output\' is missing in output dict.')
self._log_startup(input_dict, output_dict, exec_properties)
# Extract input artifacts
model_exports_uri = artifact_utils.get_single_uri(
input_dict['model_exports'])
feature_slicing_spec = evaluator_pb2.FeatureSlicingSpec()
json_format.Parse(exec_properties['feature_slicing_spec'],
feature_slicing_spec)
slice_spec = self._get_slice_spec_from_feature_slicing_spec(
feature_slicing_spec)
output_uri = artifact_utils.get_single_uri(output_dict['output'])
eval_model_path = path_utils.eval_model_path(model_exports_uri)
tf.logging.info('Using {} for model eval.'.format(eval_model_path))
eval_shared_model = tfma.default_eval_shared_model(
eval_saved_model_path=eval_model_path)
tf.logging.info('Evaluating model.')
with self._make_beam_pipeline() as pipeline:
# pylint: disable=expression-not-assigned
(pipeline
| 'ReadData' >>
beam.io.ReadFromTFRecord(file_pattern=io_utils.all_files_pattern(
artifact_utils.get_split_uri(input_dict['examples'], 'eval')))
| 'ExtractEvaluateAndWriteResults' >>
tfma.ExtractEvaluateAndWriteResults(
eval_shared_model=eval_shared_model,
slice_spec=slice_spec,
output_path=output_uri))
tf.logging.info(
'Evaluation complete. Results written to {}.'.format(output_uri))
def create_e2e_components(
pipeline_root: Text,
csv_input_location: Text,
transform_module: Text,
trainer_module: Text,
) -> List[BaseComponent]:
"""Creates components for a simple Chicago Taxi TFX pipeline for testing.
Args:
pipeline_root: The root of the pipeline output.
csv_input_location: The location of the input data directory.
transform_module: The location of the transform module file.
trainer_module: The location of the trainer module file.
Returns:
A list of TFX components that constitutes an end-to-end test pipeline.
"""
examples = dsl_utils.csv_input(csv_input_location)
example_gen = CsvExampleGen(input=examples)
statistics_gen = StatisticsGen(examples=example_gen.outputs['examples'])
infer_schema = SchemaGen(
statistics=statistics_gen.outputs['statistics'],
infer_feature_shape=False)
validate_stats = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'])
transform = Transform(
examples=example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file=transform_module)
trainer = Trainer(
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10),
eval_args=trainer_pb2.EvalArgs(num_steps=5),
module_file=trainer_module,
)
model_analyzer = Evaluator(
examples=example_gen.outputs['examples'],
model=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
model_validator = ModelValidator(
examples=example_gen.outputs['examples'], model=trainer.outputs['model'])
pusher = Pusher(
model=trainer.outputs['model'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=os.path.join(pipeline_root, 'model_serving'))))
return [
example_gen, statistics_gen, infer_schema, validate_stats, transform,
trainer, model_analyzer, model_validator, pusher
]
def _create_test_pipeline(pipeline_root: Text, csv_input_location: Text,
taxi_module_file: Text, output_bucket: Text,
enable_cache: bool):
"""Creates a simple Kubeflow-based Chicago Taxi TFX pipeline.
Args:
pipeline_name: The name of the pipeline.
pipeline_root: The root of the pipeline output.
csv_input_location: The location of the input data directory.
taxi_module_file: The location of the module file for Transform/Trainer.
enable_cache: Whether to enable cache or not.
Returns:
A logical TFX pipeline.Pipeline object.
"""
examples = csv_input(csv_input_location)
example_gen = CsvExampleGen(input_base=examples)
statistics_gen = StatisticsGen(input_data=example_gen.outputs.examples)
infer_schema = SchemaGen(stats=statistics_gen.outputs.output,
infer_feature_shape=False)
validate_stats = ExampleValidator(stats=statistics_gen.outputs.output,
schema=infer_schema.outputs.output)
transform = Transform(input_data=example_gen.outputs.examples,
schema=infer_schema.outputs.output,
module_file=taxi_module_file)
trainer = Trainer(
module_file=taxi_module_file,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
model_validator = ModelValidator(examples=example_gen.outputs.examples,
model=trainer.outputs.output)
pusher = Pusher(
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=os.path.join(output_bucket, 'model_serving'))))
return pipeline.Pipeline(
pipeline_name='chicago_taxi_pipeline_simple',
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
enable_cache=enable_cache,
)
def _create_pipeline():
"""Implements the chicago taxi pipeline with TFX."""
examples = csv_input(_data_root)
# Brings data into the pipeline or otherwise joins/converts training data.
example_gen = CsvExampleGen(input_base=examples)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(input_data=example_gen.outputs.examples)
# Generates schema based on statistics files.
infer_schema = SchemaGen(stats=statistics_gen.outputs.output)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(
stats=statistics_gen.outputs.output, schema=infer_schema.outputs.output)
# Performs transformations and feature engineering in training and serving.
transform = Transform(
input_data=example_gen.outputs.examples,
schema=infer_schema.outputs.output,
module_file=_taxi_module_file)
# Uses user-provided Python function that implements a model using TF-Learn.
trainer = Trainer(
module_file=_taxi_module_file,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(
examples=example_gen.outputs.examples, model=trainer.outputs.output)
# Checks whether the model passed the validation steps and pushes the model
# to a file destination if check passed.
pusher = Pusher(
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=_serving_model_dir)))
return [
example_gen, statistics_gen, infer_schema, validate_stats, transform,
trainer, model_analyzer, model_validator, pusher
]
def testConstructWithSliceSpec(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def test_construct_with_slice_spec(self):
examples = types.Artifact(type_name='ExamplesPath')
model_exports = types.Artifact(type_name='ModelExportPath')
evaluator = component.Evaluator(
examples=channel.as_channel([examples]),
model_exports=channel.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def testConstructWithFairnessThresholds(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]),
fairness_indicator_thresholds=[0.1, 0.3, 0.5, 0.9])
self.assertEqual('ModelEvalPath', evaluator.outputs['output'].type_name)
def _create_pipeline(pipeline_name: Text, pipeline_root: Text, data_root: Text,
module_file: Text, serving_model_dir: Text) -> pipeline.Pipeline:
examples = external_input(data_root)
input_split = example_gen_pb2.Input(splits=[
example_gen_pb2.Input.Split(name='train', pattern='train.tfrecord'),
example_gen_pb2.Input.Split(name='eval', pattern='test.tfrecord')
])
example_gen = ImportExampleGen(input_base=examples, input_config=input_split)
statistics_gen = StatisticsGen(input_data=example_gen.outputs.examples)
infer_schema = SchemaGen(stats=statistics_gen.outputs.output)
validate_stats = ExampleValidator(
stats=statistics_gen.outputs.output,
schema=infer_schema.outputs.output)
transform = Transform(
input_data=example_gen.outputs.examples,
schema=infer_schema.outputs.output,
module_file=module_file)
trainer = Trainer(
module_file=module_file,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=1000),
eval_args=trainer_pb2.EvalArgs(num_steps=500))
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec()
]))
model_validator = ModelValidator(
examples=example_gen.outputs.examples, model=trainer.outputs.output)
pusher = Pusher(
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=serving_model_dir)))
return pipeline.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats, transform,
trainer, model_analyzer, model_validator, pusher
],
additional_pipeline_args={
'tfx_image': 'tensorflow/tfx:0.14.0rc1'
},
log_root='/var/tmp/tfx/logs',
)
def testEvaluatorOnDataflowRunner(self):
"""Test for Evaluator on DataflowRunner invocation."""
pipeline_name = 'kubeflow-evaluator-dataflow-test-{}'.format(
self._random_id())
pipeline = self._create_dataflow_pipeline(pipeline_name, [
Evaluator(
examples=self._test_raw_examples,
model_exports=self._test_model,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
])),
])
self._compile_and_run_pipeline(pipeline)
def test_do(self):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
train_examples = types.TfxArtifact(type_name='ExamplesPath',
split='train')
eval_examples = types.TfxArtifact(type_name='ExamplesPath',
split='eval')
eval_examples.uri = os.path.join(source_data_dir,
'csv_example_gen/eval/')
model_exports = types.TfxArtifact(type_name='ModelExportPath')
model_exports.uri = os.path.join(source_data_dir, 'trainer/current/')
input_dict = {
'examples': [train_examples, eval_examples],
'model_exports': [model_exports],
}
# Create output dict.
eval_output = types.TfxArtifact('ModelEvalPath')
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
output_dict = {'output': [eval_output]}
# Create exec proterties.
exec_properties = {
'feature_slicing_spec':
json_format.MessageToJson(
evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour']),
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_day', 'trip_miles']),
]))
}
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
tf.gfile.Exists(os.path.join(eval_output.uri, 'eval_config')))
self.assertTrue(
tf.gfile.Exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(tf.gfile.Exists(os.path.join(eval_output.uri,
'plots')))
def testEvaluatorOnDataflowRunner(self):
"""Evaluator-only test pipeline on DataflowRunner."""
pipeline_name = 'kubeflow-evaluator-dataflow-test-{}'.format(
self._random_id())
pipeline = self._create_dataflow_pipeline(pipeline_name, [
self.raw_examples_importer, self.model_1_importer,
Evaluator(
examples=self.raw_examples_importer.outputs['result'],
model=self.model_1_importer.outputs['result'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
])
self._compile_and_run_pipeline(pipeline)
def __init__(self,
examples: types.Channel = None,
model: types.Channel = None,
feature_slicing_spec: Optional[Union[
evaluator_pb2.FeatureSlicingSpec, Dict[Text,
Any]]] = None,
fairness_indicator_thresholds: Optional[List[Union[
float, data_types.RuntimeParameter]]] = None,
output: Optional[types.Channel] = None,
model_exports: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an Evaluator component.
Args:
examples: A Channel of type `standard_artifacts.Examples`, usually
produced by an ExampleGen component. _required_
model: A Channel of type `standard_artifacts.Model`, usually produced by
a Trainer component.
feature_slicing_spec:
[evaluator_pb2.FeatureSlicingSpec](https://github.com/tensorflow/tfx/blob/master/tfx/proto/evaluator.proto)
instance that describes how Evaluator should slice the data. If any
field is provided as a RuntimeParameter, feature_slicing_spec should
be constructed as a dict with the same field names as
FeatureSlicingSpec proto message.
fairness_indicator_thresholds: Optional list of float (or
RuntimeParameter) threshold values for use with TFMA fairness
indicators. Experimental functionality: this interface and
functionality may change at any time. TODO(b/142653905): add a link to
additional documentation for TFMA fairness indicators here.
output: Channel of `ModelEvalPath` to store the evaluation results.
model_exports: Backwards compatibility alias for the `model` argument.
instance_name: Optional name assigned to this specific instance of
Evaluator. Required only if multiple Evaluator components are declared
in the same pipeline. Either `model_exports` or `model` must be present
in the input arguments.
"""
model_exports = model_exports or model
output = output or types.Channel(
type=standard_artifacts.ModelEvaluation,
artifacts=[standard_artifacts.ModelEvaluation()])
spec = EvaluatorSpec(
examples=examples,
model_exports=model_exports,
feature_slicing_spec=(feature_slicing_spec
or evaluator_pb2.FeatureSlicingSpec()),
fairness_indicator_thresholds=fairness_indicator_thresholds,
output=output)
super(Evaluator, self).__init__(spec=spec, instance_name=instance_name)
def testEvaluatorOnDataflowRunner(self):
"""Evaluator-only test pipeline on DataflowRunner."""
pipeline_name = 'kubeflow-evaluator-dataflow-test-{}'.format(
self._random_id())
pipeline = self._create_dataflow_pipeline(pipeline_name, [
Evaluator(
examples=self._input_artifacts(pipeline_name,
self._test_raw_examples),
model_exports=self._input_artifacts(pipeline_name,
self._test_model),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
])
self._compile_and_run_pipeline(pipeline)
def testDo(self):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
train_examples = standard_artifacts.Examples(split='train')
eval_examples = standard_artifacts.Examples(split='eval')
eval_examples.uri = os.path.join(source_data_dir, 'csv_example_gen/eval/')
model_exports = standard_artifacts.Model()
model_exports.uri = os.path.join(source_data_dir, 'trainer/current/')
input_dict = {
'examples': [train_examples, eval_examples],
'model_exports': [model_exports],
}
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
output_dict = {'output': [eval_output]}
# Create exec proterties.
exec_properties = {
'feature_slicing_spec':
json_format.MessageToJson(
evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour']),
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_day', 'trip_miles']),
]))
}
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
# TODO(b/141490237): Update to only check eval_config.json after TFMA
# released with corresponding change.
tf.gfile.Exists(os.path.join(eval_output.uri, 'eval_config')) or
tf.gfile.Exists(os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(tf.gfile.Exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(tf.gfile.Exists(os.path.join(eval_output.uri, 'plots')))
def __init__(self, examples: str, model_exports: str,
feature_slicing_spec: List[List[str]]):
slicing_spec = evaluator_pb2.FeatureSlicingSpec()
for slice_spec in feature_slicing_spec:
spec = slicing_spec.specs.add()
for column in slice_spec:
spec.column_for_slicing.append(column)
component = evaluator_component.Evaluator(
channel.Channel('ExamplesPath'),
channel.Channel('ModelExportPath'),
feature_slicing_spec=slicing_spec)
super().__init__(component, {
"examples": examples,
"model_exports": model_exports,
})
def __init__(self,
examples: types.Channel = None,
model: types.Channel = None,
feature_slicing_spec: Optional[
evaluator_pb2.FeatureSlicingSpec] = None,
output: Optional[types.Channel] = None,
model_exports: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an Evaluator component.
Args:
examples: A Channel of 'ExamplesPath' type, usually produced by ExampleGen
component. _required_
model: A Channel of 'ModelExportPath' type, usually produced by
Trainer component. Will be deprecated in the future for the `model`
parameter.
feature_slicing_spec:
[evaluator_pb2.FeatureSlicingSpec](https://github.com/tensorflow/tfx/blob/master/tfx/proto/evaluator.proto)
instance that describes how Evaluator should slice the data.
output: Channel of `ModelEvalPath` to store the evaluation results.
model_exports: Backwards compatibility alias for the `model`
argument.
instance_name: Optional name assigned to this specific instance of
Evaluator. Required only if multiple Evaluator components are declared
in the same pipeline.
Either `model_exports` or `model` must be present in the input arguments.
"""
model_exports = model_exports or model
output = output or types.Channel(
type=standard_artifacts.ModelEvaluation,
artifacts=[standard_artifacts.ModelEvaluation()])
spec = EvaluatorSpec(
examples=examples,
model_exports=model_exports,
feature_slicing_spec=(feature_slicing_spec
or evaluator_pb2.FeatureSlicingSpec()),
output=output)
super(Evaluator, self).__init__(spec=spec, instance_name=instance_name)
def _create_pipeline(pipeline_name: Text, pipeline_root: Text,
module_file: Text,
presto_config: presto_config_pb2.PrestoConnConfig,
query: Text, serving_model_dir: Text,
metadata_path: Text) -> pipeline.Pipeline:
"""Implements the chicago taxi pipeline with TFX."""
# Brings data into the pipeline or otherwise joins/converts training data
example_gen = PrestoExampleGen(presto_config, query=query)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(input_data=example_gen.outputs['examples'])
# Generates schema based on statistics files.
infer_schema = SchemaGen(stats=statistics_gen.outputs['output'])
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(stats=statistics_gen.outputs['output'],
schema=infer_schema.outputs['output'])
# Performs transformations and feature engineering in training and serving.
transform = Transform(input_data=example_gen.outputs['examples'],
schema=infer_schema.outputs['output'],
module_file=module_file)
# Uses user-provided Python function that implements a model using TF-Learn.
trainer = Trainer(
module_file=module_file,
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['output'],
transform_output=transform.outputs['transform_output'],
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs['examples'],
model_exports=trainer.outputs['output'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(examples=example_gen.outputs['examples'],
model=trainer.outputs['output'])
# Checks whether the model passed the validation steps and pushes the model
# to a file destination if check passed.
pusher = Pusher(model_export=trainer.outputs['output'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=serving_model_dir)))
return pipeline.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
enable_cache=True,
metadata_connection_config=metadata.sqlite_metadata_connection_config(
metadata_path),
additional_pipeline_args={},
)
def _create_pipeline(pipeline_name: Text, pipeline_root: Text, data_root: Text,
module_file: Text, serving_model_dir: Text,
metadata_path: Text) -> pipeline.Pipeline:
"""Implements the chicago taxi pipeline with TFX."""
examples = external_input(data_root)
# Brings data into the pipeline or otherwise joins/converts training data.
example_gen = CsvExampleGen(input=examples)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(examples=example_gen.outputs['examples'])
# Generates schema based on statistics files.
infer_schema = SchemaGen(statistics=statistics_gen.outputs['statistics'],
infer_feature_shape=False)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'])
# Performs transformations and feature engineering in training and serving.
transform = Transform(examples=example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file=module_file)
# Uses user-provided Python function that implements a model using TF-Learn.
trainer = Trainer(
module_file=module_file,
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs['examples'],
model_exports=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(examples=example_gen.outputs['examples'],
model=trainer.outputs['model'])
# Checks whether the model passed the validation steps and pushes the model
# to a file destination if check passed.
pusher = Pusher(model=trainer.outputs['model'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=serving_model_dir)))
return pipeline.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
enable_cache=True,
metadata_connection_config=metadata.sqlite_metadata_connection_config(
metadata_path),
additional_pipeline_args={
# LINT.IfChange
'beam_pipeline_args': [
# ----- Beam Args -----.
'--runner=PortableRunner',
# Points to the job server started in
# setup_beam_on_(flink|spark).sh
'--job_endpoint=localhost:8099',
'--environment_type=LOOPBACK',
# TODO(BEAM-6754): Utilize multicore in LOOPBACK environment. # pylint: disable=g-bad-todo
# TODO(BEAM-5167): Use concurrency information from SDK Harness. # pylint: disable=g-bad-todo
# Note; We use 100 worker threads to mitigate the issue with
# scheduling work between the Beam runner and SDK harness. Flink
# and Spark can process unlimited work items concurrently while
# SdkHarness can only process 1 work item per worker thread.
# Having 100 threads will let 100 tasks execute concurrently
# avoiding scheduling issue in most cases. In case the threads are
# exhausted, beam print the relevant message in the log.
'--experiments=worker_threads=100',
# TODO(BEAM-7199): Obviate the need for setting pre_optimize=all. # pylint: disable=g-bad-todo
'--experiments=pre_optimize=all',
# ----- Flink runner-specific Args -----.
# TODO(b/126725506): Set the task parallelism based on cpu cores.
# TODO(FLINK-10672): Obviate setting BATCH_FORCED.
'--execution_mode_for_batch=BATCH_FORCED',
],
# LINT.ThenChange(setup/setup_beam_on_spark.sh)
# LINT.ThenChange(../chicago_taxi/setup_beam_on_flink.sh)
})
class ExecutorTest(tf.test.TestCase, parameterized.TestCase):
@parameterized.named_parameters(('evaluation_w_eval_config', {
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(slicing_specs=[
tfma.SlicingSpec(feature_keys=['trip_start_hour']),
tfma.SlicingSpec(
feature_keys=['trip_start_day', 'trip_miles']),
]))
}), ('evaluation_w_module_file', {
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(slicing_specs=[
tfma.SlicingSpec(feature_keys=['trip_start_hour']),
tfma.SlicingSpec(
feature_keys=['trip_start_day', 'trip_miles']),
])),
'module_file':
None
}), ('evaluation_w_module_path', {
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(slicing_specs=[
tfma.SlicingSpec(feature_keys=['trip_start_hour']),
tfma.SlicingSpec(
feature_keys=['trip_start_day', 'trip_miles']),
])),
'module_path':
evaluator_module.__name__,
}))
def testEvalution(self, exec_properties):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
examples = standard_artifacts.Examples()
examples.uri = os.path.join(source_data_dir, 'csv_example_gen')
examples.split_names = artifact_utils.encode_split_names(['train', 'eval'])
model = standard_artifacts.Model()
baseline_model = standard_artifacts.Model()
model.uri = os.path.join(source_data_dir, 'trainer/current')
baseline_model.uri = os.path.join(source_data_dir, 'trainer/previous/')
schema = standard_artifacts.Schema()
schema.uri = os.path.join(source_data_dir, 'schema_gen')
input_dict = {
constants.EXAMPLES_KEY: [examples],
constants.MODEL_KEY: [model],
constants.SCHEMA_KEY: [schema],
}
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
blessing_output = standard_artifacts.ModelBlessing()
blessing_output.uri = os.path.join(output_data_dir, 'blessing_output')
output_dict = {
constants.EVALUATION_KEY: [eval_output],
constants.BLESSING_KEY: [blessing_output],
}
# Test multiple splits.
exec_properties[constants.EXAMPLE_SPLITS_KEY] = json_utils.dumps(
['train', 'eval'])
if 'module_file' in exec_properties:
exec_properties['module_file'] = os.path.join(source_data_dir,
'module_file',
'evaluator_module.py')
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
fileio.exists(os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'plots')))
self.assertFalse(
fileio.exists(os.path.join(blessing_output.uri, 'BLESSED')))
@parameterized.named_parameters(('legacy_feature_slicing', {
'feature_slicing_spec':
proto_utils.proto_to_json(
evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour']),
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_day', 'trip_miles']),
])),
}))
def testDoLegacySingleEvalSavedModelWFairness(self, exec_properties):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
examples = standard_artifacts.Examples()
examples.uri = os.path.join(source_data_dir, 'csv_example_gen')
examples.split_names = artifact_utils.encode_split_names(['train', 'eval'])
model = standard_artifacts.Model()
model.uri = os.path.join(source_data_dir, 'trainer/current')
input_dict = {
constants.EXAMPLES_KEY: [examples],
constants.MODEL_KEY: [model],
}
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
blessing_output = standard_artifacts.ModelBlessing()
blessing_output.uri = os.path.join(output_data_dir, 'blessing_output')
output_dict = {
constants.EVALUATION_KEY: [eval_output],
constants.BLESSING_KEY: [blessing_output],
}
try:
# Need to import the following module so that the fairness indicator
# post-export metric is registered. This may raise an ImportError if the
# currently-installed version of TFMA does not support fairness
# indicators.
import tensorflow_model_analysis.addons.fairness.post_export_metrics.fairness_indicators # pylint: disable=g-import-not-at-top, unused-variable
exec_properties['fairness_indicator_thresholds'] = [
0.1, 0.3, 0.5, 0.7, 0.9
]
except ImportError:
logging.warning(
'Not testing fairness indicators because a compatible TFMA version '
'is not installed.')
# List needs to be serialized before being passed into Do function.
exec_properties[constants.EXAMPLE_SPLITS_KEY] = json_utils.dumps(None)
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
fileio.exists(os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'plots')))
self.assertFalse(
fileio.exists(os.path.join(blessing_output.uri, 'BLESSED')))
@parameterized.named_parameters(
(
'eval_config_w_validation',
{
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(
model_specs=[
tfma.ModelSpec(label_key='tips'),
],
metrics_specs=[
tfma.MetricsSpec(metrics=[
tfma.config.MetricConfig(
class_name='ExampleCount',
# Count > 0, OK.
threshold=tfma.config.MetricThreshold(
value_threshold=tfma
.GenericValueThreshold(
lower_bound={'value': 0}))),
]),
],
slicing_specs=[tfma.SlicingSpec()]))
},
True,
True),
(
'eval_config_w_validation_fail',
{
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(
model_specs=[
tfma.ModelSpec(
name='baseline1',
label_key='tips',
is_baseline=True),
tfma.ModelSpec(
name='candidate1', label_key='tips'),
],
metrics_specs=[
tfma.MetricsSpec(metrics=[
tfma.config.MetricConfig(
class_name='ExampleCount',
# Count < -1, NOT OK.
threshold=tfma.config.MetricThreshold(
value_threshold=tfma
.GenericValueThreshold(
upper_bound={'value': -1}))),
]),
],
slicing_specs=[tfma.SlicingSpec()]))
},
False,
True),
(
'no_baseline_model_ignore_change_threshold_validation_pass',
{
'eval_config':
proto_utils.proto_to_json(
tfma.EvalConfig(
model_specs=[
tfma.ModelSpec(
name='baseline',
label_key='tips',
is_baseline=True),
tfma.ModelSpec(
name='candidate', label_key='tips'),
],
metrics_specs=[
tfma.MetricsSpec(metrics=[
tfma.config.MetricConfig(
class_name='ExampleCount',
# Count > 0, OK.
threshold=tfma.config.MetricThreshold(
value_threshold=tfma
.GenericValueThreshold(
lower_bound={'value': 0}))),
tfma.config.MetricConfig(
class_name='Accuracy',
# Should be ignored due to no baseline.
threshold=tfma.config.MetricThreshold(
change_threshold=tfma
.GenericChangeThreshold(
relative={'value': 0},
direction=tfma.MetricDirection
.LOWER_IS_BETTER))),
]),
],
slicing_specs=[tfma.SlicingSpec()]))
},
True,
False))
def testDoValidation(self, exec_properties, blessed, has_baseline):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
examples = standard_artifacts.Examples()
examples.uri = os.path.join(source_data_dir, 'csv_example_gen')
examples.split_names = artifact_utils.encode_split_names(['train', 'eval'])
model = standard_artifacts.Model()
baseline_model = standard_artifacts.Model()
model.uri = os.path.join(source_data_dir, 'trainer/current')
baseline_model.uri = os.path.join(source_data_dir, 'trainer/previous/')
blessing_output = standard_artifacts.ModelBlessing()
blessing_output.uri = os.path.join(output_data_dir, 'blessing_output')
schema = standard_artifacts.Schema()
schema.uri = os.path.join(source_data_dir, 'schema_gen')
input_dict = {
constants.EXAMPLES_KEY: [examples],
constants.MODEL_KEY: [model],
constants.SCHEMA_KEY: [schema],
}
if has_baseline:
input_dict[constants.BASELINE_MODEL_KEY] = [baseline_model]
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
blessing_output = standard_artifacts.ModelBlessing()
blessing_output.uri = os.path.join(output_data_dir, 'blessing_output')
output_dict = {
constants.EVALUATION_KEY: [eval_output],
constants.BLESSING_KEY: [blessing_output],
}
# List needs to be serialized before being passed into Do function.
exec_properties[constants.EXAMPLE_SPLITS_KEY] = json_utils.dumps(None)
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
fileio.exists(os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'plots')))
self.assertTrue(fileio.exists(os.path.join(eval_output.uri, 'validations')))
if blessed:
self.assertTrue(
fileio.exists(os.path.join(blessing_output.uri, 'BLESSED')))
else:
self.assertTrue(
fileio.exists(os.path.join(blessing_output.uri, 'NOT_BLESSED')))
class ExecutorTest(tf.test.TestCase, absl.testing.parameterized.TestCase):
# TODO(jinhuang): add test for eval_saved_model when supported.
@absl.testing.parameterized.named_parameters(('eval_config', {
'eval_config':
json_format.MessageToJson(tfma.EvalConfig(slicing_specs=[
tfma.SlicingSpec(feature_keys=['trip_start_hour']),
tfma.SlicingSpec(feature_keys=['trip_start_day', 'trip_miles']),
]),
preserving_proto_field_name=True)
}), ('eval_config_w_baseline', {
'eval_config':
json_format.MessageToJson(tfma.EvalConfig(
model_specs=[
tfma.ModelSpec(name='baseline', is_baseline=True),
tfma.ModelSpec(name='candidate'),
],
slicing_specs=[
tfma.SlicingSpec(feature_keys=['trip_start_hour']),
tfma.SlicingSpec(
feature_keys=['trip_start_day', 'trip_miles']),
]),
preserving_proto_field_name=True)
}), ('legacy_feature_slicing', {
'feature_slicing_spec':
json_format.MessageToJson(evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour']),
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_day', 'trip_miles']),
]),
preserving_proto_field_name=True),
}))
def testDo(self, exec_properties):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
examples = standard_artifacts.Examples()
examples.uri = os.path.join(source_data_dir, 'csv_example_gen')
examples.split_names = artifact_utils.encode_split_names(
['train', 'eval'])
model = standard_artifacts.Model()
baseline_model = standard_artifacts.Model()
model.uri = os.path.join(source_data_dir, 'trainer/current')
baseline_model.uri = os.path.join(source_data_dir, 'trainer/previous/')
input_dict = {
executor.EXAMPLES_KEY: [examples],
executor.MODEL_KEY: [model],
executor.BASELINE_MODEL_KEY: [baseline_model],
}
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
output_dict = {
executor.EVALUATION_KEY: [eval_output],
}
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
tf.io.gfile.exists(
os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(
tf.io.gfile.exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(
tf.io.gfile.exists(os.path.join(eval_output.uri, 'plots')))
@absl.testing.parameterized.named_parameters(('legacy_feature_slicing', {
'feature_slicing_spec':
json_format.MessageToJson(evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour']),
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_day', 'trip_miles']),
]),
preserving_proto_field_name=True),
}))
def testDoLegacySingleEvalSavedModelWFairness(self, exec_properties):
source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
# Create input dict.
examples = standard_artifacts.Examples()
examples.uri = os.path.join(source_data_dir, 'csv_example_gen')
examples.split_names = artifact_utils.encode_split_names(
['train', 'eval'])
model = standard_artifacts.Model()
baseline_model = standard_artifacts.Model()
model.uri = os.path.join(source_data_dir, 'trainer/current')
baseline_model.uri = os.path.join(source_data_dir, 'trainer/previous/')
input_dict = {
executor.EXAMPLES_KEY: [examples],
executor.MODEL_KEY: [model],
}
# Create output dict.
eval_output = standard_artifacts.ModelEvaluation()
eval_output.uri = os.path.join(output_data_dir, 'eval_output')
output_dict = {executor.EVALUATION_KEY: [eval_output]}
try:
# Need to import the following module so that the fairness indicator
# post-export metric is registered. This may raise an ImportError if the
# currently-installed version of TFMA does not support fairness
# indicators.
import tensorflow_model_analysis.addons.fairness.post_export_metrics.fairness_indicators # pylint: disable=g-import-not-at-top, unused-variable
exec_properties['fairness_indicator_thresholds'] = [
0.1, 0.3, 0.5, 0.7, 0.9
]
except ImportError:
absl.logging.warning(
'Not testing fairness indicators because a compatible TFMA version '
'is not installed.')
# Run executor.
evaluator = executor.Executor()
evaluator.Do(input_dict, output_dict, exec_properties)
# Check evaluator outputs.
self.assertTrue(
tf.io.gfile.exists(
os.path.join(eval_output.uri, 'eval_config.json')))
self.assertTrue(
tf.io.gfile.exists(os.path.join(eval_output.uri, 'metrics')))
self.assertTrue(
tf.io.gfile.exists(os.path.join(eval_output.uri, 'plots')))
def _create_pipeline(pipeline_name: Text, pipeline_root: Text,
training_data_root: Text, inference_data_root: Text,
module_file: Text,
metadata_path: Text,
direct_num_workers: int) -> pipeline.Pipeline:
"""Implements the chicago taxi pipeline with TFX."""
training_examples = external_input(training_data_root)
# Brings training data into the pipeline or otherwise joins/converts
# training data.
training_example_gen = CsvExampleGen(
input_base=training_examples, instance_name='training_example_gen')
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(
input_data=training_example_gen.outputs['examples'])
# Generates schema based on statistics files.
infer_schema = SchemaGen(
statistics=statistics_gen.outputs['statistics'],
infer_feature_shape=False)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'])
# Performs transformations and feature engineering in training and serving.
transform = Transform(
examples=training_example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file=module_file)
# Uses user-provided Python function that implements a model using TF-Learn.
trainer = Trainer(
module_file=module_file,
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000))
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=training_example_gen.outputs['examples'],
model_exports=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(
examples=training_example_gen.outputs['examples'],
model=trainer.outputs['model'])
inference_examples = external_input(inference_data_root)
# Brings inference data into the pipeline.
inference_example_gen = CsvExampleGen(
input_base=inference_examples,
output_config=example_gen_pb2.Output(
split_config=example_gen_pb2.SplitConfig(
splits=[example_gen_pb2.SplitConfig.Split(
name='unlabelled', hash_buckets=100)])),
instance_name='inference_example_gen')
# Performs offline batch inference over inference examples.
bulk_inferrer = BulkInferrer(
examples=inference_example_gen.outputs['examples'],
model=trainer.outputs['model'],
model_blessing=model_validator.outputs['blessing'],
# Empty data_spec.example_splits will result in using all splits.
data_spec=bulk_inferrer_pb2.DataSpec(),
model_spec=bulk_inferrer_pb2.ModelSpec())
return pipeline.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=[
training_example_gen, inference_example_gen, statistics_gen,
infer_schema, validate_stats, transform, trainer, model_analyzer,
model_validator, bulk_inferrer
],
enable_cache=True,
metadata_connection_config=metadata.sqlite_metadata_connection_config(
metadata_path),
# TODO(b/141578059): The multi-processing API might change.
beam_pipeline_args=['--direct_num_workers=%d' % direct_num_workers])
def _create_pipeline():
"""Implements the chicago taxi pipeline with TFX."""
query = """
SELECT
pickup_community_area,
fare,
EXTRACT(MONTH FROM trip_start_timestamp) AS trip_start_month,
EXTRACT(HOUR FROM trip_start_timestamp) AS trip_start_hour,
EXTRACT(DAYOFWEEK FROM trip_start_timestamp) AS trip_start_day,
UNIX_SECONDS(trip_start_timestamp) AS trip_start_timestamp,
pickup_latitude,
pickup_longitude,
dropoff_latitude,
dropoff_longitude,
trip_miles,
pickup_census_tract,
dropoff_census_tract,
payment_type,
company,
trip_seconds,
dropoff_community_area,
tips
FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips`
WHERE RAND() < {}""".format(_query_sample_rate)
# Brings data into the pipeline or otherwise joins/converts training data.
example_gen = BigQueryExampleGen(query=query)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(input_data=example_gen.outputs.examples)
# Generates schema based on statistics files.
infer_schema = SchemaGen(stats=statistics_gen.outputs.output)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(stats=statistics_gen.outputs.output,
schema=infer_schema.outputs.output)
# Performs transformations and feature engineering in training and serving.
transform = Transform(input_data=example_gen.outputs.examples,
schema=infer_schema.outputs.output,
module_file=_taxi_utils)
# Uses user-provided Python function that implements a model using TF-Learn.
trainer = Trainer(
module_file=_taxi_utils,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000),
custom_config={'cmle_training_args': _cmle_training_args})
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(examples=example_gen.outputs.examples,
model=trainer.outputs.output)
# Checks whether the model passed the validation steps and pushes the model
# to a file destination if check passed.
pusher = Pusher(model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
custom_config={'cmle_serving_args': _cmle_serving_args},
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=_serving_model_dir)))
return [
example_gen, statistics_gen, infer_schema, validate_stats, transform,
trainer, model_analyzer, model_validator, pusher
]
def _create_pipeline():
"""Implements the chicago taxi pipeline with TFX and Kubeflow Pipelines."""
# Brings data into the pipeline or otherwise joins/converts training data.
example_gen = BigQueryExampleGen(query=_query)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(input_data=example_gen.outputs.examples)
# Generates schema based on statistics files.
infer_schema = SchemaGen(stats=statistics_gen.outputs.output)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(stats=statistics_gen.outputs.output,
schema=infer_schema.outputs.output)
# Performs transformations and feature engineering in training and serving.
transform = Transform(input_data=example_gen.outputs.examples,
schema=infer_schema.outputs.output,
module_file=_taxi_utils)
# Uses user-provided Python function that implements a model using TF-Learn
# to train a model on Google Cloud AI Platform.
try:
from tfx.extensions.google_cloud_ai_platform.trainer import executor as ai_platform_trainer_executor # pylint: disable=g-import-not-at-top
# Train using a custom executor. This requires TFX >= 0.14.
trainer = Trainer(
executor_class=ai_platform_trainer_executor.Executor,
module_file=_taxi_utils,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000),
custom_config={
'ai_platform_training_args': _ai_platform_training_args
})
except ImportError:
# Train using a deprecated flag.
trainer = Trainer(
module_file=_taxi_utils,
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000),
custom_config={'cmle_training_args': _ai_platform_training_args})
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output,
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(examples=example_gen.outputs.examples,
model=trainer.outputs.output)
# Checks whether the model passed the validation steps and pushes the model
# to a destination if check passed.
try:
from tfx.extensions.google_cloud_ai_platform.pusher import executor as ai_platform_pusher_executor # pylint: disable=g-import-not-at-top
# Deploy the model on Google Cloud AI Platform. This requires TFX >=0.14.
pusher = Pusher(executor_class=ai_platform_pusher_executor.Executor,
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
custom_config={
'ai_platform_serving_args':
_ai_platform_serving_args
},
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=_serving_model_dir)))
except ImportError:
# Deploy the model on Google Cloud AI Platform, using a deprecated flag.
pusher = Pusher(
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
custom_config={'cmle_serving_args': _ai_platform_serving_args},
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=_serving_model_dir)))
return pipeline.Pipeline(
pipeline_name='chicago_taxi_pipeline_kubeflow',
pipeline_root=_pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
additional_pipeline_args={
'beam_pipeline_args': [
'--runner=DataflowRunner',
'--experiments=shuffle_mode=auto',
'--project=' + _project_id,
'--temp_location=' + os.path.join(_output_bucket, 'tmp'),
'--region=' + _gcp_region,
],
# Optional args:
# 'tfx_image': custom docker image to use for components.
# This is needed if TFX package is not installed from an RC
# or released version.
},
log_root='/var/tmp/tfx/logs',
)
def _create_pipeline(pipeline_name: Text, pipeline_root: Text, data_root: Text,
module_file: Text, serving_model_dir: Text,
direct_num_workers: int) -> pipeline.Pipeline:
"""Implements the chicago taxi pipeline with TFX and Kubeflow Pipelines."""
examples = external_input(data_root)
# Brings data into the pipeline or otherwise joins/converts training data.
example_gen = CsvExampleGen(input=examples)
# Computes statistics over data for visualization and example validation.
statistics_gen = StatisticsGen(examples=example_gen.outputs['examples'])
# Generates schema based on statistics files.
infer_schema = SchemaGen(statistics=statistics_gen.outputs['statistics'],
infer_feature_shape=False)
# Performs anomaly detection based on statistics and data schema.
validate_stats = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'])
# Performs transformations and feature engineering in training and serving.
transform = Transform(examples=example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file=module_file)
# Uses user-provided Python function that implements a model using TF-Learn
# to train a model on Google Cloud AI Platform.
trainer = Trainer(
module_file=module_file,
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10000),
eval_args=trainer_pb2.EvalArgs(num_steps=5000),
)
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs['examples'],
model_exports=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(examples=example_gen.outputs['examples'],
model=trainer.outputs['model'])
# Checks whether the model passed the validation steps and pushes the model
# to Google Cloud AI Platform if check passed.
pusher = Pusher(model=trainer.outputs['model'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=serving_model_dir)))
return pipeline.Pipeline(
pipeline_name=pipeline_name,
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
# TODO(b/141578059): The multi-processing API might change.
beam_pipeline_args=['--direct_num_workers=%d' % direct_num_workers],
additional_pipeline_args={},
)
def _create_pipeline(pipeline_root: Text,
csv_input_location: data_types.RuntimeParameter,
taxi_module_file: data_types.RuntimeParameter,
enable_cache: bool):
"""Creates a simple Kubeflow-based Chicago Taxi TFX pipeline.
Args:
pipeline_root: The root of the pipeline output.
csv_input_location: The location of the input data directory.
taxi_module_file: The location of the module file for Transform/Trainer.
enable_cache: Whether to enable cache or not.
Returns:
A logical TFX pipeline.Pipeline object.
"""
examples = external_input(csv_input_location)
example_gen = CsvExampleGen(input=examples)
statistics_gen = StatisticsGen(examples=example_gen.outputs['examples'])
infer_schema = SchemaGen(
statistics=statistics_gen.outputs['statistics'],
infer_feature_shape=False,
)
validate_stats = ExampleValidator(
statistics=statistics_gen.outputs['statistics'],
schema=infer_schema.outputs['schema'],
)
transform = Transform(
examples=example_gen.outputs['examples'],
schema=infer_schema.outputs['schema'],
module_file=taxi_module_file,
)
trainer = Trainer(
module_file=taxi_module_file,
transformed_examples=transform.outputs['transformed_examples'],
schema=infer_schema.outputs['schema'],
transform_graph=transform.outputs['transform_graph'],
train_args=trainer_pb2.TrainArgs(num_steps=10),
eval_args=trainer_pb2.EvalArgs(num_steps=5),
)
model_analyzer = Evaluator(
examples=example_gen.outputs['examples'],
model=trainer.outputs['model'],
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]),
)
model_validator = ModelValidator(examples=example_gen.outputs['examples'],
model=trainer.outputs['model'])
pusher = Pusher(
model=trainer.outputs['model'],
model_blessing=model_validator.outputs['blessing'],
push_destination=pusher_pb2.PushDestination(
filesystem=pusher_pb2.PushDestination.Filesystem(
base_directory=os.path.join(str(pipeline.ROOT_PARAMETER),
'model_serving'))),
)
return pipeline.Pipeline(
pipeline_name='parameterized_tfx_oss',
pipeline_root=pipeline_root,
components=[
example_gen, statistics_gen, infer_schema, validate_stats,
transform, trainer, model_analyzer, model_validator, pusher
],
enable_cache=enable_cache,
)
def __init__(
self,
examples: types.Channel = None,
model: types.Channel = None,
baseline_model: Optional[types.Channel] = None,
# TODO(b/148618405): deprecate feature_slicing_spec.
feature_slicing_spec: Optional[Union[
evaluator_pb2.FeatureSlicingSpec, Dict[Text, Any]]] = None,
fairness_indicator_thresholds: Optional[List[Union[
float, data_types.RuntimeParameter]]] = None,
example_splits: Optional[List[Text]] = None,
output: Optional[types.Channel] = None,
model_exports: Optional[types.Channel] = None,
instance_name: Optional[Text] = None,
eval_config: Optional[tfma.EvalConfig] = None,
blessing: Optional[types.Channel] = None,
schema: Optional[types.Channel] = None,
module_file: Optional[Text] = None,
module_path: Optional[Text] = None):
"""Construct an Evaluator component.
Args:
examples: A Channel of type `standard_artifacts.Examples`, usually
produced by an ExampleGen component. _required_
model: A Channel of type `standard_artifacts.Model`, usually produced by
a Trainer component.
baseline_model: An optional channel of type 'standard_artifacts.Model' as
the baseline model for model diff and model validation purpose.
feature_slicing_spec:
Deprecated, please use eval_config instead. Only support estimator.
[evaluator_pb2.FeatureSlicingSpec](https://github.com/tensorflow/tfx/blob/master/tfx/proto/evaluator.proto)
instance that describes how Evaluator should slice the data. If any
field is provided as a RuntimeParameter, feature_slicing_spec should
be constructed as a dict with the same field names as
FeatureSlicingSpec proto message.
fairness_indicator_thresholds: Optional list of float (or
RuntimeParameter) threshold values for use with TFMA fairness
indicators. Experimental functionality: this interface and
functionality may change at any time. TODO(b/142653905): add a link
to additional documentation for TFMA fairness indicators here.
example_splits: Names of splits on which the metrics are computed.
Default behavior (when example_splits is set to None or Empty) is using
the 'eval' split.
output: Channel of `ModelEvaluation` to store the evaluation results.
model_exports: Backwards compatibility alias for the `model` argument.
instance_name: Optional name assigned to this specific instance of
Evaluator. Required only if multiple Evaluator components are declared
in the same pipeline. Either `model_exports` or `model` must be present
in the input arguments.
eval_config: Instance of tfma.EvalConfig containg configuration settings
for running the evaluation. This config has options for both estimator
and Keras.
blessing: Output channel of 'ModelBlessing' that contains the
blessing result.
schema: A `Schema` channel to use for TFXIO.
module_file: A path to python module file containing UDFs for Evaluator
customization. The module_file can implement following functions at its
top level.
def custom_eval_shared_model(
eval_saved_model_path, model_name, eval_config, **kwargs,
) -> tfma.EvalSharedModel:
def custom_extractors(
eval_shared_model, eval_config, tensor_adapter_config,
) -> List[tfma.extractors.Extractor]:
module_path: A python path to the custom module that contains the UDFs.
See 'module_file' for the required signature of UDFs. Note this can
not be set together with module_file.
"""
if bool(module_file) and bool(module_path):
raise ValueError(
'Python module path can not be set together with module file path.'
)
if eval_config is not None and feature_slicing_spec is not None:
raise ValueError(
"Exactly one of 'eval_config' or 'feature_slicing_spec' "
"must be supplied.")
if eval_config is None and feature_slicing_spec is None:
feature_slicing_spec = evaluator_pb2.FeatureSlicingSpec()
logging.info(
'Neither eval_config nor feature_slicing_spec is passed, '
'the model is treated as estimator.')
if model_exports:
logging.warning(
'The "model_exports" argument to the Evaluator component has '
'been renamed to "model" and is deprecated. Please update your '
'usage as support for this argument will be removed soon.')
model = model_exports
if feature_slicing_spec:
logging.warning('feature_slicing_spec is deprecated, please use '
'eval_config instead.')
blessing = blessing or types.Channel(
type=standard_artifacts.ModelBlessing)
evaluation = output or types.Channel(
type=standard_artifacts.ModelEvaluation)
spec = EvaluatorSpec(
examples=examples,
model=model,
baseline_model=baseline_model,
feature_slicing_spec=feature_slicing_spec,
fairness_indicator_thresholds=fairness_indicator_thresholds,
example_splits=json_utils.dumps(example_splits),
evaluation=evaluation,
eval_config=eval_config,
blessing=blessing,
schema=schema,
module_file=module_file,
module_path=module_path)
super(Evaluator, self).__init__(spec=spec, instance_name=instance_name)
