def notebook_pipeline():
"""A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill."""
from kfp_notebook.pipeline import NotebookOp
notebook_op = NotebookOp(name="${name}",
notebook="${notebook}",
cos_endpoint="${cos_endpoint}",
cos_bucket="${cos_bucket}",
cos_directory="${cos_directory}",
cos_dependencies_archive="${cos_dependencies_archive}",
requirements_url="${requirements_url}",
image="${image}")
from kubernetes.client.models import V1EnvVar
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}"))
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}"))
from kfp import onprem
notebook_op.container.add_env_variable(V1EnvVar(name='DATA_DIR', value="${mount_path}"))
notebook_op.apply(onprem.mount_pvc(pvc_name='${dataset_pvc}',
volume_name='${dataset_pvc}',
volume_mount_path='${mount_path}'))
def test_construct_with_env_variables_tekton():
notebook_op = NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_envs={
"ENV_VAR_ONE": "1",
"ENV_VAR_TWO": "2",
"ENV_VAR_THREE": "3"
},
image="test/image:dev",
workflow_engine="Tekton")
confirmation_names = [
"ENV_VAR_ONE", "ENV_VAR_TWO", "ENV_VAR_THREE", "ELYRA_RUN_NAME"
]
confirmation_values = ["1", "2", "3"]
field_path = "metadata.annotations['pipelines.kubeflow.org/run_name']"
for env_val in notebook_op.container.env:
assert env_val.name in confirmation_names
confirmation_names.remove(env_val.name)
if env_val.name == 'ELYRA_RUN_NAME':
assert env_val.value_from.field_ref.field_path == field_path, env_val.value_from.field_ref
else:
assert env_val.value in confirmation_values
confirmation_values.remove(env_val.value)
# Verify confirmation values have been drained.
assert len(confirmation_names) == 0
assert len(confirmation_values) == 0
def test_construct_with_env_variables():
notebook_op = NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_envs={
"ENV_VAR_ONE": "1",
"ENV_VAR_TWO": "2",
"ENV_VAR_THREE": "3"
},
image="test/image:dev")
confirmation_names = ["ENV_VAR_ONE", "ENV_VAR_TWO", "ENV_VAR_THREE"]
confirmation_values = ["1", "2", "3"]
for env_val in notebook_op.container.env:
assert env_val.name in confirmation_names
assert env_val.value in confirmation_values
confirmation_names.remove(env_val.name)
confirmation_values.remove(env_val.value)
# Verify confirmation values have been drained.
assert len(confirmation_names) == 0
assert len(confirmation_values) == 0
def notebook_op():
return NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
def test_fail_without_name():
with pytest.raises(TypeError):
NotebookOp(notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
def test_fail_without_runtime_image():
with pytest.raises(ValueError) as error_info:
NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz")
assert "You need to provide an image." == str(error_info.value)
def test_properly_set_python_script_name_when_in_subdirectory():
notebook_op = NotebookOp(name="test",
notebook="foo/test.py",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
assert "test.py" == notebook_op.notebook_name
def test_override_bootstrap_url():
notebook_op = NotebookOp(name="test",
bootstrap_script_url="https://test.server.com/bootscript.py",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
assert notebook_op.bootstrap_script_url == "https://test.server.com/bootscript.py"
def test_override_requirements_url():
notebook_op = NotebookOp(name="test",
requirements_url="https://test.server.com/requirements.py",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
assert notebook_op.requirements_url == "https://test.server.com/requirements.py"
def test_fail_without_notebook():
with pytest.raises(TypeError):
NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
def test_fail_with_empty_string_as_notebook():
with pytest.raises(ValueError) as error_info:
NotebookOp(name="test",
notebook="",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
assert "You need to provide a notebook." == str(error_info.value)
def test_construct_with_bad_pipeline_outputs():
with pytest.raises(ValueError) as error_info:
NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_outputs=['test_output1.txt', 'test;output2.txt'],
image="test/image:dev")
assert "Illegal character (;) found in filename 'test;output2.txt'." == str(error_info.value)
def test_user_volume_size():
notebook_op = NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev",
emptydir_volume_size='20Gi')
assert notebook_op.emptydir_volume_size == '20Gi'
assert notebook_op.container_work_dir_root_path == '/mnt/'
def test_fail_with_empty_string_as_name():
with pytest.raises(ValueError):
NotebookOp(name="",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
def test_properly_set_notebook_name_when_in_subdirectory():
notebook_op = NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="foo/test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev")
assert "test_notebook.ipynb" == notebook_op.notebook_name
def test_construct_with_only_pipeline_outputs():
notebook_op = NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_outputs=['test_output1.txt', 'test,output2.txt'],
pipeline_envs={},
image="test/image:dev")
assert notebook_op.pipeline_outputs == ['test_output1.txt', 'test,output2.txt']
assert '--outputs "test_output1.txt;test,output2.txt"' in notebook_op.container.args[0]
def test_user_crio_volume_creation():
notebook_op = NotebookOp(name="test",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev",
emptydir_volume_size='20Gi')
assert notebook_op.emptydir_volume_size == '20Gi'
assert notebook_op.container_work_dir_root_path == '/opt/app-root/src/'
assert notebook_op.container.volume_mounts.__len__() == 1
assert notebook_op.container.env.__len__() == 1
def test_user_crio_volume_creation():
notebook_op = NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
image="test/image:dev",
emptydir_volume_size='20Gi')
assert notebook_op.emptydir_volume_size == '20Gi'
assert notebook_op.container_work_dir_root_path == '/opt/app-root/src/'
assert notebook_op.container.volume_mounts.__len__() == 1
# Environment variables: PYTHONPATH, ELYRA_RUN_NAME
assert notebook_op.container.env.__len__() == 2, notebook_op.container.env
def run_notebook_op(op_name):
notebook_op = NotebookOp(name=op_name,
notebook='Untitled.ipynb',
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=cos_dependencies_archive,
pipeline_outputs=outputs,
pipeline_inputs=inputs,
image=image)
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_ACCESS_KEY_ID', value=cos_username))
notebook_op.container.add_env_variable(
V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value=cos_password))
notebook_op.container.set_image_pull_policy('Always')
return notebook_op
def notebook_pipeline():
"""A pipeline to run a Jupyter notebook with elyra-ai/kfp-notebook and Papermill."""
from kfp_notebook.pipeline import NotebookOp
notebook_op = NotebookOp(name="${name}",
notebook="${notebook}",
cos_endpoint="${cos_endpoint}",
cos_bucket="${cos_bucket}",
cos_directory="${cos_directory}",
cos_dependencies_archive="${cos_dependencies_archive}",
requirements_url="${requirements_url}",
image="${image}")
from kubernetes.client.models import V1EnvVar
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_ACCESS_KEY_ID', value="${cos_username}"))
notebook_op.container.add_env_variable(V1EnvVar(name='AWS_SECRET_ACCESS_KEY', value="${cos_password}"))
notebook_op.container.set_image_pull_policy('Always')
def test_construct_wiildcard_outputs():
notebook_op = NotebookOp(
name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_inputs=['test_input1.txt', 'test_input2.txt'],
pipeline_outputs=['test_out*', 'foo.tar'],
image="test/image:dev")
assert notebook_op.pipeline_inputs == [
'test_input1.txt', 'test_input2.txt'
]
assert notebook_op.pipeline_outputs == ['test_out*', 'foo.tar']
assert '--inputs "test_input1.txt;test_input2.txt"' in notebook_op.container.args[
0]
assert '--outputs "test_out*;foo.tar"' in notebook_op.container.args[0]
def _cc_pipeline(self,
pipeline,
pipeline_name,
pipeline_version='',
experiment_name='',
cos_directory=None,
export=False):
runtime_configuration = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
cos_secret = runtime_configuration.metadata.get('cos_secret')
if cos_directory is None:
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
self.log_pipeline_info(pipeline_name,
f"processing pipeline dependencies to: {cos_endpoint} "
f"bucket: {cos_bucket} folder: {cos_directory}")
t0_all = time.time()
emptydir_volume_size = ''
container_runtime = bool(os.getenv('CRIO_RUNTIME', 'False').lower() == 'true')
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# Sort operations based on dependency graph (topological order)
sorted_operations = PipelineProcessor._sort_operations(pipeline.operations)
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
PipelineProcessor._propagate_operation_inputs_outputs(pipeline, sorted_operations)
for operation in sorted_operations:
operation_artifact_archive = self._get_dependency_archive_name(operation)
self.log.debug("Creating pipeline component :\n {op} archive : {archive}".format(
op=operation, archive=operation_artifact_archive))
if container_runtime:
# Volume size to create when using CRI-o, NOTE: IBM Cloud minimum is 20Gi
emptydir_volume_size = '20Gi'
# Collect env variables
pipeline_envs = self._collect_envs(operation,
cos_secret=cos_secret,
cos_username=cos_username,
cos_password=cos_password)
# Include any envs set on the operation
pipeline_envs.update(operation.env_vars_as_dict(logger=self.log))
sanitized_operation_name = self._sanitize_operation_name(operation.name)
# create pipeline operation
notebook_ops[operation.id] = NotebookOp(name=sanitized_operation_name,
pipeline_name=pipeline_name,
experiment_name=experiment_name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
pipeline_version=pipeline_version,
pipeline_source=pipeline.source,
pipeline_inputs=operation.inputs,
pipeline_outputs=operation.outputs,
pipeline_envs=pipeline_envs,
emptydir_volume_size=emptydir_volume_size,
cpu_request=operation.cpu,
mem_request=operation.memory,
gpu_limit=operation.gpu,
image=operation.runtime_image,
file_outputs={
'mlpipeline-metrics':
'{}/mlpipeline-metrics.json'
.format(pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR']),
'mlpipeline-ui-metadata':
'{}/mlpipeline-ui-metadata.json'
.format(pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'])
})
if cos_secret and not export:
notebook_ops[operation.id].apply(use_aws_secret(cos_secret))
image_namespace = self._get_metadata_configuration(namespace=MetadataManager.NAMESPACE_RUNTIME_IMAGES)
for image_instance in image_namespace:
if image_instance.metadata['image_name'] == operation.runtime_image and \
image_instance.metadata.get('pull_policy'):
notebook_ops[operation.id].container.set_image_pull_policy(image_instance.metadata['pull_policy'])
self.log_pipeline_info(pipeline_name,
f"processing operation dependencies for id: {operation.id}",
operation_name=operation.name)
self._upload_dependencies_to_object_store(runtime_configuration,
cos_directory,
operation)
# Process dependencies after all the operations have been created
for operation in pipeline.operations.values():
op = notebook_ops[operation.id]
for parent_operation_id in operation.parent_operations:
parent_op = notebook_ops[parent_operation_id] # Parent Operation
op.after(parent_op)
self.log_pipeline_info(pipeline_name, "pipeline dependencies processed", duration=(time.time() - t0_all))
return notebook_ops
def _cc_pipeline(self, pipeline, pipeline_name):
runtime_configuration = self._get_runtime_configuration(pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
self.log_pipeline_info(pipeline_name,
f"processing pipeline dependencies to: {cos_endpoint} "
f"bucket: {cos_bucket} folder: {pipeline_name}")
t0_all = time.time()
emptydir_volume_size = ''
container_runtime = bool(os.getenv('CRIO_RUNTIME', 'False').lower() == 'true')
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
for operation in pipeline.operations.values():
parent_io = [] # gathers inputs & outputs relative to parent
for parent_operation_id in operation.parent_operations:
parent_operation = pipeline.operations[parent_operation_id]
if parent_operation.inputs:
parent_io.extend(parent_operation.inputs)
if parent_operation.outputs:
parent_io.extend(parent_operation.outputs)
if parent_io:
operation.inputs = parent_io
for operation in pipeline.operations.values():
operation_artifact_archive = self._get_dependency_archive_name(operation)
self.log.debug("Creating pipeline component :\n {op} archive : {archive}".format(
op=operation, archive=operation_artifact_archive))
if container_runtime:
# Volume size to create when using CRI-o, NOTE: IBM Cloud minimum is 20Gi
emptydir_volume_size = '20Gi'
# Collect env variables
pipeline_envs = dict()
pipeline_envs['AWS_ACCESS_KEY_ID'] = cos_username
pipeline_envs['AWS_SECRET_ACCESS_KEY'] = cos_password
# Convey pipeline logging enablement to operation
pipeline_envs['ELYRA_ENABLE_PIPELINE_INFO'] = str(self.enable_pipeline_info)
# Setting identifies a writable directory in the container image.
# Only Unix-style path spec is supported.
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'] = self.WCD
if operation.env_vars:
for env_var in operation.env_vars:
# Strip any of these special characters from both key and value
# Splits on the first occurrence of '='
result = [x.strip(' \'\"') for x in env_var.split('=', 1)]
# Should be non empty key with a value
if len(result) == 2 and result[0] != '':
pipeline_envs[result[0]] = result[1]
# create pipeline operation
notebook_ops[operation.id] = NotebookOp(name=operation.name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
pipeline_inputs=operation.inputs,
pipeline_outputs=operation.outputs,
pipeline_envs=pipeline_envs,
emptydir_volume_size=emptydir_volume_size,
image=operation.runtime_image,
file_outputs={
'mlpipeline-metrics':
'{}/mlpipeline-metrics.json'
.format(pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR']),
'mlpipeline-ui-metadata':
'{}/mlpipeline-ui-metadata.json'
.format(pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'])
})
self.log_pipeline_info(pipeline_name,
f"processing operation dependencies for id: {operation.id}",
operation_name=operation.name)
# upload operation dependencies to object storage
try:
t0 = time.time()
dependency_archive_path = self._generate_dependency_archive(operation)
self.log_pipeline_info(pipeline_name,
f"generated dependency archive: {dependency_archive_path}",
operation_name=operation.name,
duration=(time.time() - t0))
cos_client = CosClient(config=runtime_configuration)
t0 = time.time()
cos_client.upload_file_to_dir(dir=cos_directory,
file_name=operation_artifact_archive,
file_path=dependency_archive_path)
self.log_pipeline_info(pipeline_name,
f"uploaded dependency archive to: {cos_directory}/{operation_artifact_archive}",
operation_name=operation.name,
duration=(time.time() - t0))
except FileNotFoundError as ex:
self.log.error("Dependencies were not found building archive for operation: {}".
format(operation.name), exc_info=True)
raise FileNotFoundError("Node '{}' referenced dependencies that were not found: {}".
format(operation.name, ex))
except BaseException as ex:
self.log.error("Error uploading artifacts to object storage for operation: {}".
format(operation.name), exc_info=True)
raise ex from ex
# Process dependencies after all the operations have been created
for operation in pipeline.operations.values():
op = notebook_ops[operation.id]
for parent_operation_id in operation.parent_operations:
parent_op = notebook_ops[parent_operation_id] # Parent Operation
op.after(parent_op)
self.log_pipeline_info(pipeline_name, "pipeline dependencies processed", duration=(time.time() - t0_all))
return notebook_ops
def _cc_pipeline(self,
pipeline,
pipeline_name,
pipeline_version='',
experiment_name='',
cos_directory=None):
runtime_configuration = self._get_metadata_configuration(
namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
if cos_directory is None:
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
self.log_pipeline_info(
pipeline_name,
f"processing pipeline dependencies to: {cos_endpoint} "
f"bucket: {cos_bucket} folder: {cos_directory}")
t0_all = time.time()
emptydir_volume_size = ''
container_runtime = bool(
os.getenv('CRIO_RUNTIME', 'False').lower() == 'true')
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# Sort operations based on dependency graph (topological order)
sorted_operations = PipelineProcessor._sort_operations(
pipeline.operations)
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
PipelineProcessor._propagate_operation_inputs_outputs(
pipeline, sorted_operations)
for operation in sorted_operations:
operation_artifact_archive = self._get_dependency_archive_name(
operation)
self.log.debug(
"Creating pipeline component :\n {op} archive : {archive}".
format(op=operation, archive=operation_artifact_archive))
if container_runtime:
# Volume size to create when using CRI-o, NOTE: IBM Cloud minimum is 20Gi
emptydir_volume_size = '20Gi'
# Collect env variables
pipeline_envs = dict()
pipeline_envs['AWS_ACCESS_KEY_ID'] = cos_username
pipeline_envs['AWS_SECRET_ACCESS_KEY'] = cos_password
# Convey pipeline logging enablement to operation
pipeline_envs['ELYRA_ENABLE_PIPELINE_INFO'] = str(
self.enable_pipeline_info)
# Setting identifies a writable directory in the container image.
# Only Unix-style path spec is supported.
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'] = self.WCD
if operation.env_vars:
for env_var in operation.env_vars:
# Strip any of these special characters from both key and value
# Splits on the first occurrence of '='
result = [x.strip(' \'\"') for x in env_var.split('=', 1)]
# Should be non empty key with a value
if len(result) == 2 and result[0] != '':
pipeline_envs[result[0]] = result[1]
sanitized_operation_name = self._sanitize_operation_name(
operation.name)
# create pipeline operation
notebook_ops[operation.id] = NotebookOp(
name=sanitized_operation_name,
pipeline_name=pipeline_name,
experiment_name=experiment_name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
pipeline_version=pipeline_version,
pipeline_source=pipeline.source,
pipeline_inputs=operation.inputs,
pipeline_outputs=operation.outputs,
pipeline_envs=pipeline_envs,
emptydir_volume_size=emptydir_volume_size,
cpu_request=operation.cpu,
mem_request=operation.memory,
gpu_limit=operation.gpu,
image=operation.runtime_image,
file_outputs={
'mlpipeline-metrics':
'{}/mlpipeline-metrics.json'.format(
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR']),
'mlpipeline-ui-metadata':
'{}/mlpipeline-ui-metadata.json'.format(
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'])
})
image_namespace = self._get_metadata_configuration(
namespace=MetadataManager.NAMESPACE_RUNTIME_IMAGES)
for image_instance in image_namespace:
if image_instance.metadata['image_name'] == operation.runtime_image and \
image_instance.metadata.get('pull_policy'):
notebook_ops[operation.id].container.set_image_pull_policy(
image_instance.metadata['pull_policy'])
self.log_pipeline_info(
pipeline_name,
f"processing operation dependencies for id: {operation.id}",
operation_name=operation.name)
self._upload_dependencies_to_object_store(runtime_configuration,
cos_directory, operation)
# Process dependencies after all the operations have been created
for operation in pipeline.operations.values():
op = notebook_ops[operation.id]
for parent_operation_id in operation.parent_operations:
parent_op = notebook_ops[
parent_operation_id] # Parent Operation
op.after(parent_op)
self.log_pipeline_info(pipeline_name,
"pipeline dependencies processed",
duration=(time.time() - t0_all))
return notebook_ops
def _cc_pipeline(self,
pipeline,
pipeline_name,
pipeline_version='',
experiment_name='',
cos_directory=None,
export=False):
runtime_configuration = self._get_metadata_configuration(
namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
cos_secret = runtime_configuration.metadata.get('cos_secret')
if cos_directory is None:
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
self.log_pipeline_info(
pipeline_name,
f"processing pipeline dependencies to: {cos_endpoint} "
f"bucket: {cos_bucket} folder: {cos_directory}")
t0_all = time.time()
emptydir_volume_size = ''
container_runtime = bool(
os.getenv('CRIO_RUNTIME', 'False').lower() == 'true')
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# Sort operations based on dependency graph (topological order)
sorted_operations = PipelineProcessor._sort_operations(
pipeline.operations)
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
PipelineProcessor._propagate_operation_inputs_outputs(
pipeline, sorted_operations)
for operation in sorted_operations:
if container_runtime:
# Volume size to create when using CRI-o, NOTE: IBM Cloud minimum is 20Gi
emptydir_volume_size = '20Gi'
# Collect env variables
pipeline_envs = self._collect_envs(operation,
cos_secret=cos_secret,
cos_username=cos_username,
cos_password=cos_password)
sanitized_operation_name = self._sanitize_operation_name(
operation.name)
# Create pipeline operation
# If operation is one of the "standard" set of NBs or scripts, construct custom NotebookOp
if operation.classifier in [
"execute-notebook-node", "execute-python-node",
"execute-r-node"
]:
operation_artifact_archive = self._get_dependency_archive_name(
operation)
self.log.debug(
"Creating pipeline component :\n {op} archive : {archive}".
format(op=operation, archive=operation_artifact_archive))
notebook_ops[operation.id] = NotebookOp(
name=sanitized_operation_name,
pipeline_name=pipeline_name,
experiment_name=experiment_name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
pipeline_version=pipeline_version,
pipeline_source=pipeline.source,
pipeline_inputs=operation.inputs,
pipeline_outputs=operation.outputs,
pipeline_envs=pipeline_envs,
emptydir_volume_size=emptydir_volume_size,
cpu_request=operation.cpu,
mem_request=operation.memory,
gpu_limit=operation.gpu,
image=operation.runtime_image,
file_outputs={
'mlpipeline-metrics':
'{}/mlpipeline-metrics.json'.format(
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR']),
'mlpipeline-ui-metadata':
'{}/mlpipeline-ui-metadata.json'.format(
pipeline_envs['ELYRA_WRITABLE_CONTAINER_DIR'])
})
# TODO Can we move all of this to apply to non-standard components as well? Test when servers are up
if cos_secret and not export:
notebook_ops[operation.id].apply(
use_aws_secret(cos_secret))
image_namespace = self._get_metadata_configuration(
namespace=MetadataManager.NAMESPACE_RUNTIME_IMAGES)
for image_instance in image_namespace:
if image_instance.metadata['image_name'] == operation.runtime_image and \
image_instance.metadata.get('pull_policy'):
notebook_ops[operation.id].container. \
set_image_pull_policy(image_instance.metadata['pull_policy'])
self.log_pipeline_info(
pipeline_name,
f"processing operation dependencies for id: {operation.id}",
operation_name=operation.name)
self._upload_dependencies_to_object_store(
runtime_configuration, cos_directory, operation)
# If operation is a "non-standard" component, load it's spec and create operation with factory function
else:
component_source = {}
component_source[
operation.
component_source_type] = operation.component_source
# Build component task factory
try:
factory_function = components.load_component(
**component_source)
except Exception:
# TODO Fix error messaging and break exceptions down into categories
self.log.error(
f"There was an error while loading component spec for {operation.name}."
)
raise RuntimeError(
f"There was an error while loading component spec for {operation.name}."
)
# Add factory function, which returns a ContainerOp task instance, to pipeline operation dict
try:
notebook_ops[operation.id] = factory_function(
**operation.component_params)
except Exception:
# TODO Fix error messaging and break exceptions down into categories
self.log.error(
f"There was an error while constructing component {operation.name}."
)
raise RuntimeError(
f"There was an error while constructing component {operation.name}."
)
# Process dependencies after all the operations have been created
for operation in pipeline.operations.values():
op = notebook_ops[operation.id]
for parent_operation_id in operation.parent_operations:
parent_op = notebook_ops[
parent_operation_id] # Parent Operation
op.after(parent_op)
self.log_pipeline_info(pipeline_name,
"pipeline dependencies processed",
duration=(time.time() - t0_all))
return notebook_ops
def test_normalize_label_value():
valid_middle_chars = '-_.'
# test min length
assert NotebookOp._normalize_label_value(None) == ''
assert NotebookOp._normalize_label_value('') == ''
# test max length (63)
assert NotebookOp._normalize_label_value('a' * 63) ==\
'a' * 63
assert NotebookOp._normalize_label_value('a' * 64) ==\
'a' * 63 # truncated
# test first and last char
assert NotebookOp._normalize_label_value('1') == '1'
assert NotebookOp._normalize_label_value('22') == '22'
assert NotebookOp._normalize_label_value('3_3') == '3_3'
assert NotebookOp._normalize_label_value('4u4') == '4u4'
assert NotebookOp._normalize_label_value('5$5') == '5_5'
# test first char
for c in string.printable:
if c in string.ascii_letters + string.digits:
# first char is valid
# no length violation
assert NotebookOp._normalize_label_value(c) == c
assert NotebookOp._normalize_label_value(c + 'B') == c + 'B'
# max length
assert NotebookOp._normalize_label_value(c + 'B' * 62) ==\
(c + 'B' * 62)
# max length exceeded
assert NotebookOp._normalize_label_value(c + 'B' * 63) ==\
(c + 'B' * 62) # truncated
else:
# first char is invalid, e.g. '#a', and becomes the
# second char, which might require replacement
rv = c
if c not in valid_middle_chars:
rv = '_'
# no length violation
assert NotebookOp._normalize_label_value(c) == 'a' + rv + 'a'
assert NotebookOp._normalize_label_value(c + 'B') == 'a' + rv + 'B'
# max length
assert NotebookOp._normalize_label_value(c + 'B' * 62) ==\
('a' + rv + 'B' * 61) # truncated
# max length exceeded
assert NotebookOp._normalize_label_value(c + 'B' * 63) ==\
('a' + rv + 'B' * 61) # truncated
# test last char
for c in string.printable:
if c in string.ascii_letters + string.digits:
# no length violation
assert NotebookOp._normalize_label_value('b' + c) == 'b' + c
# max length
assert NotebookOp._normalize_label_value('b' * 62 + c) ==\
('b' * 62 + c)
# max length exceeded
assert NotebookOp._normalize_label_value('b' * 63 + c) ==\
('b' * 63)
else:
# last char is invalid, e.g. 'a#', and requires
# patching
rv = c
if c not in valid_middle_chars:
rv = '_'
# no length violation (char is appended)
assert NotebookOp._normalize_label_value('b' + c) == 'b' + rv + 'a'
# max length (char is replaced)
assert NotebookOp._normalize_label_value('b' * 62 + c) ==\
('b' * 62 + 'a')
# max length exceeded (no action required)
assert NotebookOp._normalize_label_value('b' * 63 + c) ==\
('b' * 63)
# test first and last char
for c in string.printable:
if c in string.ascii_letters + string.digits:
# no length violation
assert NotebookOp._normalize_label_value(c + 'b' + c) ==\
c + 'b' + c # nothing is modified
# max length
assert NotebookOp._normalize_label_value(c + 'b' * 61 + c) ==\
(c + 'b' * 61 + c) # nothing is modified
# max length exceeded
assert NotebookOp._normalize_label_value(c + 'b' * 62 + c) ==\
c + 'b' * 62 # truncate only
else:
# first and last characters are invalid, e.g. '#a#'
rv = c
if c not in valid_middle_chars:
rv = '_'
# no length violation
assert NotebookOp._normalize_label_value(c + 'b' + c) ==\
'a' + rv + 'b' + rv + 'a'
# max length
assert NotebookOp._normalize_label_value(c + 'b' * 59 + c) ==\
('a' + rv + 'b' * 59 + rv + 'a')
# max length exceeded after processing, scenario 1
# resolved by adding char before first, replace last
assert NotebookOp._normalize_label_value(c + 'b' * 60 + c) ==\
('a' + rv + 'b' * 60 + 'a')
# max length exceeded after processing, scenario 2
# resolved by adding char before first, appending after last
assert NotebookOp._normalize_label_value(c + 'b' * 59 + c) ==\
('a' + rv + 'b' * 59 + rv + 'a')
# max length exceeded before processing, scenario 1
# resolved by adding char before first, truncating last
assert NotebookOp._normalize_label_value(c + 'b' * 62 + c) ==\
('a' + rv + 'b' * 61)
# max length exceeded before processing, scenario 2
# resolved by adding char before first, replacing last
assert NotebookOp._normalize_label_value(c + 'b' * 60 + c * 3) ==\
('a' + rv + 'b' * 60 + 'a')
# test char in a position other than first and last
# if invalid, the char is replaced with '_'
for c in string.printable:
if c in string.ascii_letters + string.digits + '-_.':
assert NotebookOp._normalize_label_value('A' + c + 'Z') ==\
'A' + c + 'Z'
else:
assert NotebookOp._normalize_label_value('A' + c + 'Z') == 'A_Z'
# encore
assert NotebookOp._normalize_label_value(r'¯\_(ツ)_/¯') == 'a_________a'
def _cc_pipeline(self, pipeline, pipeline_name):
runtime_configuration = self._get_runtime_configuration(
pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
emptydir_volume_size = ''
container_runtime = bool(
os.getenv('CRIO_RUNTIME', 'False').lower() == 'true')
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
for operation in pipeline.operations.values():
parent_io = [] # gathers inputs & outputs relative to parent
for parent_operation_id in operation.parent_operations:
parent_operation = pipeline.operations[parent_operation_id]
if parent_operation.inputs:
parent_io.extend(parent_operation.inputs)
if parent_operation.outputs:
parent_io.extend(parent_operation.outputs)
if parent_io:
operation.inputs = parent_io
for operation in pipeline.operations.values():
operation_artifact_archive = self._get_dependency_archive_name(
operation)
self.log.debug(
"Creating pipeline component :\n {op} archive : {archive}".
format(op=operation, archive=operation_artifact_archive))
if container_runtime:
# Volume size to create when using CRI-o, NOTE: IBM Cloud minimum is 20Gi
emptydir_volume_size = '20Gi'
# Collect env variables
pipeline_envs = dict()
pipeline_envs['AWS_ACCESS_KEY_ID'] = cos_username
pipeline_envs['AWS_SECRET_ACCESS_KEY'] = cos_password
if operation.env_vars:
for env_var in operation.env_vars:
# Strip any of these special characters from both key and value
# Splits on the first occurrence of '='
result = [x.strip(' \'\"') for x in env_var.split('=', 1)]
# Should be non empty key with a value
if len(result) == 2 and result[0] != '':
pipeline_envs[result[0]] = result[1]
# create pipeline operation
notebook_ops[operation.id] = NotebookOp(
name=operation.name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
pipeline_inputs=operation.inputs,
pipeline_outputs=operation.outputs,
pipeline_envs=pipeline_envs,
emptydir_volume_size=emptydir_volume_size,
image=operation.runtime_image)
self.log.info("NotebookOp Created for Component '%s' (%s)",
operation.name, operation.id)
# upload operation dependencies to object storage
try:
t0 = time.time()
dependency_archive_path = self._generate_dependency_archive(
operation)
t1 = time.time()
self.log.debug(
"Generation of dependency archive for operation '{name}' took {duration:.3f} secs."
.format(name=operation.name, duration=(t1 - t0)))
cos_client = CosClient(config=runtime_configuration)
t0 = time.time()
cos_client.upload_file_to_dir(
dir=cos_directory,
file_name=operation_artifact_archive,
file_path=dependency_archive_path)
t1 = time.time()
self.log.debug(
"Upload of dependency archive for operation '{name}' took {duration:.3f} secs."
.format(name=operation.name, duration=(t1 - t0)))
except FileNotFoundError as ex:
self.log.error(
"Dependencies were not found building archive for operation: {}"
.format(operation.name),
exc_info=True)
raise FileNotFoundError(
"Node '{}' referenced dependencies that were not found: {}"
.format(operation.name, ex))
except BaseException as ex:
self.log.error(
"Error uploading artifacts to object storage for operation: {}"
.format(operation.name),
exc_info=True)
raise ex from ex
self.log.info(
"Pipeline dependencies have been uploaded to object storage")
# Process dependencies after all the operations have been created
for operation in pipeline.operations.values():
op = notebook_ops[operation.id]
for parent_operation_id in operation.parent_operations:
parent_op = notebook_ops[
parent_operation_id] # Parent Operation
op.after(parent_op)
return notebook_ops
def test_construct_with_env_variables_argo():
notebook_op = NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_envs={
"ENV_VAR_ONE": "1",
"ENV_VAR_TWO": "2",
"ENV_VAR_THREE": "3"
},
image="test/image:dev")
confirmation_names = [
"ENV_VAR_ONE", "ENV_VAR_TWO", "ENV_VAR_THREE", "ELYRA_RUN_NAME"
]
confirmation_values = [
"1", "2", "3",
"{{workflow.annotations.pipelines.kubeflow.org/run_name}}"
]
for env_val in notebook_op.container.env:
assert env_val.name in confirmation_names
assert env_val.value in confirmation_values
confirmation_names.remove(env_val.name)
confirmation_values.remove(env_val.value)
# Verify confirmation values have been drained.
assert len(confirmation_names) == 0
assert len(confirmation_values) == 0
# same as before but explicitly specify the workflow engine type
# as Argo
notebook_op = NotebookOp(name="test",
pipeline_name="test-pipeline",
experiment_name="experiment-name",
notebook="test_notebook.ipynb",
cos_endpoint="http://testserver:32525",
cos_bucket="test_bucket",
cos_directory="test_directory",
cos_dependencies_archive="test_archive.tgz",
pipeline_envs={
"ENV_VAR_ONE": "1",
"ENV_VAR_TWO": "2",
"ENV_VAR_THREE": "3"
},
image="test/image:dev",
workflow_engine="Argo")
confirmation_names = [
"ENV_VAR_ONE", "ENV_VAR_TWO", "ENV_VAR_THREE", "ELYRA_RUN_NAME"
]
confirmation_values = [
"1", "2", "3",
"{{workflow.annotations.pipelines.kubeflow.org/run_name}}"
]
for env_val in notebook_op.container.env:
assert env_val.name in confirmation_names
assert env_val.value in confirmation_values
confirmation_names.remove(env_val.name)
confirmation_values.remove(env_val.value)
# Verify confirmation values have been drained.
assert len(confirmation_names) == 0
assert len(confirmation_values) == 0
def _cc_pipeline(self, pipeline, pipeline_name):
runtime_configuration = self._get_runtime_configuration(pipeline.runtime_config)
cos_endpoint = runtime_configuration.metadata['cos_endpoint']
cos_username = runtime_configuration.metadata['cos_username']
cos_password = runtime_configuration.metadata['cos_password']
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata['cos_bucket']
# Create dictionary that maps component Id to its ContainerOp instance
notebook_ops = {}
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
for pipeline_operation in pipeline.operations.values():
parent_inputs_and_outputs = []
for parent_operation_id in pipeline_operation.parent_operations:
parent_operation = pipeline.operations[parent_operation_id]
if parent_operation.inputs:
parent_inputs_and_outputs.extend(parent_operation.inputs)
if parent_operation.outputs:
parent_inputs_and_outputs.extend(parent_operation.outputs)
if parent_inputs_and_outputs:
pipeline_operation.inputs = parent_inputs_and_outputs
for operation in pipeline.operations.values():
operation_artifact_archive = self._get_dependency_archive_name(operation)
self.log.debug("Creating pipeline component :\n "
"componentID : %s \n "
"name : %s \n "
"parent_operations : %s \n "
"dependencies : %s \n "
"dependencies include subdirectories : %s \n "
"filename : %s \n "
"archive : %s \n "
"inputs : %s \n "
"outputs : %s \n "
"runtime image : %s \n ",
operation.id,
operation.name,
operation.parent_operations,
operation.dependencies,
operation.include_subdirectories,
operation.filename,
operation_artifact_archive,
operation.inputs,
operation.outputs,
operation.runtime_image)
# create pipeline operation
notebook_op = NotebookOp(name=operation.name,
notebook=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
image=operation.runtime_image)
if operation.inputs:
notebook_op.add_pipeline_inputs(self._artifact_list_to_str(operation.inputs))
if operation.outputs:
notebook_op.add_pipeline_outputs(self._artifact_list_to_str(operation.outputs))
notebook_op.add_environment_variable('AWS_ACCESS_KEY_ID', cos_username)
notebook_op.add_environment_variable('AWS_SECRET_ACCESS_KEY', cos_password)
# Set ENV variables
if operation.env_vars:
for env_var in operation.env_vars:
# Strip any of these special characters from both key and value
# Splits on the first occurrence of '='
result = [x.strip(' \'\"') for x in env_var.split('=', 1)]
# Should be non empty key with a value
if len(result) == 2 and result[0] != '':
notebook_op.add_environment_variable(result[0], result[1])
notebook_ops[operation.id] = notebook_op
self.log.info("NotebookOp Created for Component '%s' (%s)", operation.name, operation.id)
# upload operation dependencies to object storage
try:
dependency_archive_path = self._generate_dependency_archive(operation)
cos_client = CosClient(config=runtime_configuration)
cos_client.upload_file_to_dir(dir=cos_directory,
file_name=operation_artifact_archive,
file_path=dependency_archive_path)
except BaseException:
self.log.error("Error uploading artifacts to object storage.", exc_info=True)
raise
self.log.info("Pipeline dependencies have been uploaded to object storage")
# Process dependencies after all the operations have been created
for pipeline_operation in pipeline.operations.values():
op = notebook_ops[pipeline_operation.id]
for parent_operation_id in pipeline_operation.parent_operations:
parent_op = notebook_ops[parent_operation_id] # Parent Operation
op.after(parent_op)
return notebook_ops
