def test_taxi_pipeline_check_dag_construction(self):
airflow_config = {
'schedule_interval': None,
'start_date': datetime.datetime(2019, 1, 1),
}
logical_pipeline = taxi_pipeline_portable_beam._create_pipeline()
self.assertEqual(9, len(logical_pipeline.components))
TfxRunner(airflow_config).run(logical_pipeline)
def test_taxi_pipeline_check_dag_construction(self):
airflow_config = {
'schedule_interval': None,
'start_date': datetime.datetime(2019, 1, 1),
}
logical_pipeline = taxi_pipeline_simple._create_pipeline()
self.assertEqual(9, len(logical_pipeline.components))
pipeline = TfxRunner(airflow_config).run(logical_pipeline)
self.assertIsInstance(pipeline, models.DAG)
transformed_examples=transform.outputs.transformed_examples,
schema=infer_schema.outputs.output,
transform_output=transform.outputs.transform_output,
train_steps=10000,
eval_steps=5000,
warm_starting=True)
# Uses TFMA to compute a evaluation statistics over features of a model.
model_analyzer = Evaluator(
examples=example_gen.outputs.examples,
model_exports=trainer.outputs.output)
# Performs quality validation of a candidate model (compared to a baseline).
model_validator = ModelValidator(
examples=example_gen.outputs.examples, model=trainer.outputs.output)
# Checks whether the model passed the validation steps and pushes the model
# to a file destination if check passed.
pusher = Pusher(
model_export=trainer.outputs.output,
model_blessing=model_validator.outputs.blessing,
serving_model_dir=serving_model_dir)
return [
example_gen, statistics_gen, infer_schema, validate_stats, transform,
trainer, model_analyzer, model_validator, pusher
]
pipeline = TfxRunner(airflow_config).run(create_pipeline())