def deploy_to_dev(pipeline):
"""
Deploy the Kubeflow Pipelines Pipeline (e.g. a method decorated with `@dsl.pipeline`)
to Kubeflow and execute it.
Args:
pipeline (func): The `@dsl.pipeline` method describing the pipeline
Returns:
True if the deployment suceeds
"""
deploy_args = dict()
pipeline_name = pipeline.__name__
experiment_name = f"{pipeline_name}_tests"
run_name = pipeline_name + ' ' + datetime.now().strftime(
'%Y-%m-%d %H-%M-%S')
print(f"hm> pipeline: {pipeline_name}")
print(f"hm> experiment: {experiment_name}")
print(f"hm> run: {run_name}")
client = Client(None, None)
client.create_run_from_pipeline_func(pipeline,
deploy_args,
run_name=run_name,
experiment_name=experiment_name)
print(f"hm> Deployed and running!")
return True
def test_pipelines(name: str, params: list, fn: Callable):
"""Runs each pipeline that it's been parameterized for, and waits for it to succeed."""
client = Client('127.0.0.1:8888')
run = client.create_run_from_pipeline_func(
fn, arguments={p['name']: p['value']
for p in params})
completed = client.wait_for_run_completion(run.run_id, timeout=1200)
status = completed.to_dict()['run']['status']
assert status == 'Succeeded', f'Pipeline status is {status}'
def test_pipelines():
status = json.loads(
check_output([
"microk8s",
"kubectl",
"get",
"services/ml-pipeline",
"-nkubeflow",
"-ojson",
]).decode("utf-8"))
ip = status["spec"]["clusterIP"]
client = Client(f"http://{ip}:8888")
run = client.create_run_from_pipeline_func(
download_and_join,
arguments={
"url1":
"gs://ml-pipeline/sample-data/shakespeare/shakespeare1.txt",
"url2":
"gs://ml-pipeline/sample-data/shakespeare/shakespeare2.txt",
},
)
completed = client.wait_for_run_completion(run.run_id, timeout=3600)
status = completed.to_dict()["run"]["status"]
assert status == "Succeeded", f"Pipeline cowsay status is {status}"
def run_pipeline(
pipeline,
arguments=None,
project=None,
experiment=None,
run=None,
namespace=None,
artifact_path=None,
ops=None,
url=None,
ttl=None,
):
"""remote KubeFlow pipeline execution
Submit a workflow task to KFP via mlrun API service
:param pipeline: KFP pipeline function or path to .yaml/.zip pipeline file
:param arguments: pipeline arguments
:param experiment: experiment name
:param run: optional, run name
:param namespace: Kubernetes namespace (if not using default)
:param url: optional, url to mlrun API service
:param artifact_path: target location/url for mlrun artifacts
:param ops: additional operators (.apply() to all pipeline functions)
:param ttl: pipeline ttl in secs (after that the pods will be removed)
:returns: kubeflow pipeline id
"""
remote = not get_k8s_helper(
silent=True).is_running_inside_kubernetes_cluster()
artifact_path = artifact_path or mlconf.artifact_path
if artifact_path and "{{run.uid}}" in artifact_path:
artifact_path.replace("{{run.uid}}", "{{workflow.uid}}")
if artifact_path and "{{run.project}}" in artifact_path:
if not project:
raise ValueError("project name must be specified with this" +
f" artifact_path template {artifact_path}")
artifact_path.replace("{{run.project}}", project)
if not artifact_path:
raise ValueError("artifact path was not specified")
namespace = namespace or mlconf.namespace
arguments = arguments or {}
if remote or url:
mldb = get_run_db(url)
if mldb.kind != "http":
raise ValueError(
"run pipeline require access to remote api-service"
", please set the dbpath url")
id = mldb.submit_pipeline(
pipeline,
arguments,
experiment=experiment,
run=run,
namespace=namespace,
ops=ops,
artifact_path=artifact_path,
)
else:
client = Client(namespace=namespace)
if isinstance(pipeline, str):
experiment = client.create_experiment(name=experiment)
run_result = client.run_pipeline(experiment.id,
run,
pipeline,
params=arguments)
else:
conf = new_pipe_meta(artifact_path, ttl, ops)
run_result = client.create_run_from_pipeline_func(
pipeline,
arguments,
run_name=run,
experiment_name=experiment,
pipeline_conf=conf,
)
id = run_result.run_id
logger.info("Pipeline run id={}, check UI or DB for progress".format(id))
return id
class KubeflowClient(object):
log = logging.getLogger(__name__)
def __init__(self, config, project_name, context):
token = AuthHandler().obtain_id_token()
self.host = config.host
self.client = Client(self.host, existing_token=token)
self.project_name = project_name
self.pipeline_description = config.run_config.description
self.generator = PipelineGenerator(config, project_name, context)
def list_pipelines(self):
pipelines = self.client.list_pipelines(page_size=30).pipelines
return tabulate(map(lambda x: [x.name, x.id], pipelines),
headers=["Name", "ID"])
def run_once(
self,
pipeline,
image,
experiment_name,
run_name,
wait,
image_pull_policy="IfNotPresent",
) -> None:
run = self.client.create_run_from_pipeline_func(
self.generator.generate_pipeline(pipeline, image,
image_pull_policy),
arguments={},
experiment_name=experiment_name,
run_name=run_name,
)
if wait:
run.wait_for_run_completion(timeout=WAIT_TIMEOUT)
def compile(self,
pipeline,
image,
output,
image_pull_policy="IfNotPresent"):
Compiler().compile(
self.generator.generate_pipeline(pipeline, image,
image_pull_policy),
output,
)
self.log.info("Generated pipeline definition was saved to %s" % output)
def upload(self, pipeline, image, image_pull_policy="IfNotPresent"):
pipeline = self.generator.generate_pipeline(pipeline, image,
image_pull_policy)
if self._pipeline_exists(self.project_name):
pipeline_id = self._get_pipeline_id(self.project_name)
version_id = self._upload_pipeline_version(pipeline, pipeline_id)
self.log.info("New version of pipeline created: %s", version_id)
else:
(pipeline_id, version_id) = self._upload_pipeline(pipeline)
self.log.info("Pipeline created")
self.log.info(
f"Pipeline link: {self.host}/#/pipelines/details/%s/version/%s",
pipeline_id,
version_id,
)
def _pipeline_exists(self, pipeline_name):
return self._get_pipeline_id(pipeline_name) is not None
def _get_pipeline_id(self, pipeline_name):
pipelines = self.client.pipelines.list_pipelines(filter=json.dumps({
"predicates": [{
"key": "name",
"op": 1,
"string_value": pipeline_name,
}]
})).pipelines
if pipelines:
return pipelines[0].id
def _upload_pipeline_version(self, pipeline_func, pipeline_id):
version_name = f"{clean_name(self.project_name)}-{uuid.uuid4()}"[:100]
with NamedTemporaryFile(suffix=".yaml") as f:
Compiler().compile(pipeline_func, f.name)
return self.client.pipeline_uploads.upload_pipeline_version(
f.name,
name=version_name,
pipelineid=pipeline_id,
_request_timeout=10000,
).id
def _upload_pipeline(self, pipeline_func):
with NamedTemporaryFile(suffix=".yaml") as f:
Compiler().compile(pipeline_func, f.name)
pipeline = self.client.pipeline_uploads.upload_pipeline(
f.name,
name=self.project_name,
description=self.pipeline_description,
_request_timeout=10000,
)
return (pipeline.id, pipeline.default_version.id)
def _ensure_experiment_exists(self, experiment_name):
try:
experiment = self.client.get_experiment(
experiment_name=experiment_name)
self.log.info(f"Existing experiment found: {experiment.id}")
except ValueError as e:
if not str(e).startswith("No experiment is found"):
raise
experiment = self.client.create_experiment(experiment_name)
self.log.info(f"New experiment created: {experiment.id}")
return experiment.id
def schedule(self, experiment_name, cron_expression):
experiment_id = self._ensure_experiment_exists(experiment_name)
pipeline_id = self._get_pipeline_id(self.project_name)
self._disable_runs(experiment_id, pipeline_id)
self.client.create_recurring_run(
experiment_id,
f"{self.project_name} on {cron_expression}",
cron_expression=cron_expression,
pipeline_id=pipeline_id,
)
self.log.info("Pipeline scheduled to %s", cron_expression)
def _disable_runs(self, experiment_id, pipeline_id):
runs = self.client.list_recurring_runs(experiment_id=experiment_id)
if runs.jobs is not None:
my_runs = [
job for job in runs.jobs
if job.pipeline_spec.pipeline_id == pipeline_id
]
for job in my_runs:
self.client.jobs.delete_job(job.id)
self.log.info(f"Previous schedule deleted {job.id}")
def run_pipeline(pipeline,
arguments=None,
experiment=None,
run=None,
namespace=None,
artifact_path=None,
ops=None,
url=None,
remote=False):
"""remote KubeFlow pipeline execution
Submit a workflow task to KFP via mlrun API service
:param pipeline KFP pipeline function or path to .yaml/.zip pipeline file
:param arguments pipeline arguments
:param experiment experiment name
:param run optional, run name
:param namespace Kubernetes namespace (if not using default)
:param url optional, url to mlrun API service
:param artifact_path target location/url for mlrun artifacts
:param ops additional operators (.apply() to all pipeline functions)
:param remote use mlrun remote API service vs direct KFP APIs
:return kubeflow pipeline id
"""
namespace = namespace or mlconf.namespace
arguments = arguments or {}
if remote or url:
mldb = get_run_db(url).connect()
if mldb.kind != 'http':
raise ValueError(
'run pipeline require access to remote api-service'
', please set the dbpath url')
id = mldb.submit_pipeline(pipeline,
arguments,
experiment=experiment,
run=run,
namespace=namespace,
ops=ops,
artifact_path=artifact_path)
else:
client = Client(namespace=namespace)
if isinstance(pipeline, str):
experiment = client.create_experiment(name=experiment)
run_result = client.run_pipeline(experiment.id,
run,
pipeline,
params=arguments)
else:
conf = new_pipe_meta(artifact_path, ops)
run_result = client.create_run_from_pipeline_func(
pipeline,
arguments,
run_name=run,
experiment_name=experiment,
pipeline_conf=conf)
id = run_result.run_id
logger.info('Pipeline run id={}, check UI or DB for progress'.format(id))
return id
split_data_op = components.create_component_from_func(split_data,
base_image=BASE_IMAGE)
train_op = components.create_component_from_func(train, base_image=BASE_IMAGE)
@dsl.pipeline(name=PIPELINE_NAME,
description='test classifier pipeline with breast cancer dataset'
)
def pipeline(test_size: float = 0.2):
fetch_data_task = fetch_data_op()
split_data_task = split_data_op(x=fetch_data_task.outputs['x'],
y=fetch_data_task.outputs['y'],
test_size=test_size)
# TODO: train model(s) (with tuning) in parallel?
train_task = train_op(x=split_data_task.outputs['x_train'],
y=split_data_task.outputs['y_train'])
# TODO: batch predictions (move to separate pipeline)
# TODO: serve model
if __name__ == '__main__':
client = Client(host=CLIENT)
client.create_run_from_pipeline_func(
pipeline,
arguments={},
run_name=create_version_name(),
experiment_name=f'{EXPERIMENT_NAME}_dev')
# TODO: github action to compile & deploy pipeline on release
# TODO: unit tests on commit
# TODO: connect artifacts to local storage mount
class KubeflowClient(object):
log = logging.getLogger(__name__)
def __init__(self, config, project_name, context):
token = self.obtain_id_token()
self.host = config.host
self.client = Client(self.host, existing_token=token)
self.project_name = project_name
self.context = context
dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = True
self.volume_meta = config.run_config.volume
def list_pipelines(self):
pipelines = self.client.list_pipelines(page_size=30).pipelines
return tabulate(map(lambda x: [x.name, x.id], pipelines),
headers=["Name", "ID"])
def run_once(
self,
pipeline,
image,
experiment_name,
run_name,
wait,
image_pull_policy="IfNotPresent",
) -> None:
run = self.client.create_run_from_pipeline_func(
self.generate_pipeline(pipeline, image, image_pull_policy),
arguments={},
experiment_name=experiment_name,
run_name=run_name,
)
if wait:
run.wait_for_run_completion(timeout=WAIT_TIMEOUT)
def obtain_id_token(self):
from google.auth.transport.requests import Request
from google.oauth2 import id_token
from google.auth.exceptions import DefaultCredentialsError
client_id = os.environ.get(IAP_CLIENT_ID, None)
jwt_token = None
if not client_id:
self.log.info(
"No IAP_CLIENT_ID provided, skipping custom IAP authentication"
)
return jwt_token
try:
self.log.debug("Obtaining JWT token for %s." + client_id)
jwt_token = id_token.fetch_id_token(Request(), client_id)
self.log.info("Obtained JWT token for MLFLOW connectivity.")
except DefaultCredentialsError as ex:
self.log.warning(
str(ex) +
(" Note that this authentication method does not work with default"
" credentials obtained via 'gcloud auth application-default login'"
" command. Refer to documentation on how to configure service account"
" locally"
" (https://cloud.google.com/docs/authentication/production#manually)"
))
except Exception as e:
self.log.error("Failed to obtain IAP access token. " + str(e))
finally:
return jwt_token
def generate_pipeline(self, pipeline, image, image_pull_policy):
@dsl.pipeline(
name=self.project_name,
description="Kubeflow pipeline for Kedro project",
)
def convert_kedro_pipeline_to_kfp() -> None:
"""Convert from a Kedro pipeline into a kfp container graph."""
node_volumes = (_setup_volumes()
if self.volume_meta is not None else {})
node_dependencies = self.context.pipelines.get(
pipeline).node_dependencies
kfp_ops = _build_kfp_ops(node_dependencies, node_volumes)
for node, dependencies in node_dependencies.items():
for dependency in dependencies:
kfp_ops[node.name].after(kfp_ops[dependency.name])
def _setup_volumes():
vop = dsl.VolumeOp(
name="data-volume-create",
resource_name="data-volume",
size=self.volume_meta.size,
modes=self.volume_meta.access_modes,
storage_class=self.volume_meta.storageclass,
)
if self.volume_meta.skip_init:
return {"/home/kedro/data": vop.volume}
else:
volume_init = dsl.ContainerOp(
name="data-volume-init",
image=image,
command=["sh", "-c"],
arguments=[
" ".join([
"cp",
"--verbose",
"-r",
"/home/kedro/data/*",
"/home/kedro/datavolume",
])
],
pvolumes={"/home/kedro/datavolume": vop.volume},
)
volume_init.container.set_image_pull_policy(image_pull_policy)
return {"/home/kedro/data": volume_init.pvolume}
def _build_kfp_ops(node_dependencies: Dict[Node, Set[Node]],
node_volumes: Dict) -> Dict[str, dsl.ContainerOp]:
"""Build kfp container graph from Kedro node dependencies. """
kfp_ops = {}
env = [
V1EnvVar(name=IAP_CLIENT_ID,
value=os.environ.get(IAP_CLIENT_ID, ""))
]
if is_mlflow_enabled():
kfp_ops["mlflow-start-run"] = dsl.ContainerOp(
name="mlflow-start-run",
image=image,
command=["kedro"],
arguments=[
"kubeflow",
"mlflow-start",
dsl.RUN_ID_PLACEHOLDER,
],
file_outputs={"mlflow_run_id": "/tmp/mlflow_run_id"},
)
kfp_ops["mlflow-start-run"].container.set_image_pull_policy(
image_pull_policy)
env.append(
V1EnvVar(
name="MLFLOW_RUN_ID",
value=kfp_ops["mlflow-start-run"].output,
))
for node in node_dependencies:
name = _clean_name(node.name)
kfp_ops[node.name] = dsl.ContainerOp(
name=name,
image=image,
command=["kedro"],
arguments=["run", "--node", node.name],
pvolumes=node_volumes,
container_kwargs={"env": env},
)
kfp_ops[node.name].container.set_image_pull_policy(
image_pull_policy)
return kfp_ops
return convert_kedro_pipeline_to_kfp
def compile(self,
pipeline,
image,
output,
image_pull_policy="IfNotPresent"):
Compiler().compile(
self.generate_pipeline(pipeline, image, image_pull_policy), output)
self.log.info("Generated pipeline definition was saved to %s" % output)
def upload(self, pipeline, image, image_pull_policy="IfNotPresent"):
pipeline = self.generate_pipeline(pipeline, image, image_pull_policy)
if self._pipeline_exists(self.project_name):
pipeline_id = self._get_pipeline_id(self.project_name)
version_id = self._upload_pipeline_version(pipeline, pipeline_id,
self.project_name)
self.log.info("New version of pipeline created: %s", version_id)
else:
(pipeline_id,
version_id) = self._upload_pipeline(pipeline, self.project_name)
self.log.info("Pipeline created")
self.log.info(
f"Pipeline link: {self.host}/#/pipelines/details/%s/version/%s",
pipeline_id,
version_id,
)
def _pipeline_exists(self, pipeline_name):
return self._get_pipeline_id(pipeline_name) is not None
def _get_pipeline_id(self, pipeline_name):
pipelines = self.client.pipelines.list_pipelines(filter=json.dumps({
"predicates": [{
"key": "name",
"op": 1,
"string_value": pipeline_name,
}]
})).pipelines
if pipelines:
return pipelines[0].id
def _upload_pipeline_version(self, pipeline_func, pipeline_id,
pipeline_name):
version_name = f"{_clean_name(pipeline_name)}-{uuid.uuid4()}"[:100]
with NamedTemporaryFile(suffix=".yaml") as f:
Compiler().compile(pipeline_func, f.name)
return self.client.pipeline_uploads.upload_pipeline_version(
f.name, name=version_name, pipelineid=pipeline_id).id
def _upload_pipeline(self, pipeline_func, pipeline_name):
with NamedTemporaryFile(suffix=".yaml") as f:
Compiler().compile(pipeline_func, f.name)
pipeline = self.client.pipeline_uploads.upload_pipeline(
f.name, name=pipeline_name)
return (pipeline.id, pipeline.default_version.id)
def _ensure_experiment_exists(self, experiment_name):
try:
experiment = self.client.get_experiment(
experiment_name=experiment_name)
self.log.info(f"Existing experiment found: {experiment.id}")
except ValueError as e:
if not str(e).startswith("No experiment is found"):
raise
experiment = self.client.create_experiment(experiment_name)
self.log.info(f"New experiment created: {experiment.id}")
return experiment.id
def schedule(self, experiment_name, cron_expression):
experiment_id = self._ensure_experiment_exists(experiment_name)
pipeline_id = self._get_pipeline_id(self.project_name)
self._disable_runs(experiment_id, pipeline_id)
self.client.create_recurring_run(
experiment_id,
f"{self.project_name} on {cron_expression}",
cron_expression=cron_expression,
pipeline_id=pipeline_id,
)
self.log.info("Pipeline scheduled to %s", cron_expression)
def _disable_runs(self, experiment_id, pipeline_id):
runs = self.client.list_recurring_runs(experiment_id=experiment_id)
if runs.jobs is not None:
my_runs = [
job for job in runs.jobs
if job.pipeline_spec.pipeline_id == pipeline_id
]
for job in my_runs:
self.client.jobs.delete_job(job.id)
self.log.info(f"Previous schedule deleted {job.id}")
from kfp import Client as KfpClient
import pipeline
client = KfpClient(
host=
"https://kubeflow-oct.endpoints.myfirstproject-226013.cloud.goog/pipeline",
client_id=
"478111835512-94l3kca44ueiudgm9tr2ibg4ihah5g3d.apps.googleusercontent.com")
client.create_pipeline_version(
pipeline_id="1234",
name="$COMMIT_SHA",
repo_name="$REPO_NAME",
commit_sha="$COMMIT_SHA",
url="https://storage.googleapis.com/$REPO_NAME/$COMMIT_SHA/pipeline.zip")
'''
client = KfpClient(
host="https://kubeflow-oct.endpoints.myfirstproject-226013.cloud.goog/pipeline",
client_id="478111835512-94l3kca44ueiudgm9tr2ibg4ihah5g3d.apps.googleusercontent.com"
)
pipeline_func = pipeline.preprocessing_pl
client.create_run_from_pipeline_func(
pipeline_func=pipeline_func,
arguments={'data_path': 'gs://kubeflow_pipelines_sentiment/data/test.csv',
'vectorizer_gcs_location': 'gs://kubeflow_pipelines_sentiment/assets/x.pkl'},
experiment_name='test2',
run_name='001'
)'''
