def pipeline_ml_obj():
full_pipeline = Pipeline(
[
node(
func=preprocess_fun,
inputs="raw_data",
outputs="data",
tags=["inference", "training"],
),
node(func=fit_fun, inputs="data", outputs="model", tags=["training"]),
node(
func=predict_fun,
inputs=["data", "model"],
outputs="predictions",
tags=["inference"],
),
]
)
pipeline_ml_obj = pipeline_ml(
training=full_pipeline.only_nodes_with_tags("training"),
inference=full_pipeline.only_nodes_with_tags("inference"),
input_name="raw_data",
)
return pipeline_ml_obj
def dummy_pipeline_ml(dummy_pipeline):
dummy_pipeline_ml = pipeline_ml(
training=dummy_pipeline.only_nodes_with_tags("training"),
inference=dummy_pipeline.only_nodes_with_tags("inference"),
input_name="raw_data",
)
return dummy_pipeline_ml
def pipeline_ml_with_tag(pipeline_with_tag):
pipeline_ml_with_tag = pipeline_ml(
training=pipeline_with_tag,
inference=Pipeline([
node(func=predict_fun,
inputs=["model", "data"],
outputs="predictions")
]),
input_name="data",
)
return pipeline_ml_with_tag
def test_too_many_free_inputs():
with pytest.raises(KedroMlflowPipelineMLInputsError,
match="No free input is allowed"):
pipeline_ml(
training=Pipeline([
node(
func=preprocess_fun,
inputs="raw_data",
outputs="neither_data_nor_model",
)
]),
inference=Pipeline([
node(
func=predict_fun,
inputs=["model", "data"],
outputs="predictions",
)
]),
input_name="data",
)
def pipeline_ml_with_intermediary_artifacts():
full_pipeline = Pipeline([
node(
func=preprocess_fun,
inputs="raw_data",
outputs="data",
tags=["training"],
),
node(
func=fit_encoder_fun,
inputs="data",
outputs="encoder",
tags=["training"],
),
node(
func=apply_encoder_fun,
inputs=["encoder", "data"],
outputs="encoded_data",
tags=["training", "inference"],
),
node(
func=train_fun,
inputs="encoded_data",
outputs="model",
tags=["training"],
),
node(
func=predict_fun,
inputs=["model", "encoded_data"],
outputs="predictions",
tags=["inference"],
),
])
pipeline_ml_with_tag = pipeline_ml(
training=full_pipeline.only_nodes_with_tags("training"),
inference=full_pipeline.only_nodes_with_tags("inference"),
input_name="data",
)
return pipeline_ml_with_tag
def create_pipelines(**kwargs) -> Dict[str, Pipeline]:
"""Create the project's pipeline.
Args:
kwargs: Ignore any additional arguments added in the future.
Returns:
A mapping from a pipeline name to a ``Pipeline`` object.
"""
preprocessing_pipeline = pipeline.create_pipeline().only_nodes_with_tags(
"preprocessing")
training_pipeline = pipeline.create_pipeline().only_nodes_with_tags(
"training")
evaluation_pipeline = pipeline.create_pipeline().only_nodes_with_tags(
"evaluation")
inference_pipeline = pipeline.create_pipeline().only_nodes_with_tags(
"inference")
kedro_mlflow_pipeline = pipeline_ml(
training=preprocessing_pipeline + training_pipeline +
evaluation_pipeline,
inference=inference_pipeline,
input_name="features",
)
return {
"preprocessing":
preprocessing_pipeline,
"training":
training_pipeline,
"evaluation":
evaluation_pipeline,
"kedro_mlflow":
kedro_mlflow_pipeline,
"__default__":
preprocessing_pipeline + training_pipeline + evaluation_pipeline,
}