def _run_compiled_code(self, script_path: str):
_name = self.pipeline.config.pipeline_name
pipeline_yaml_path = kfputils.compile_pipeline(script_path, _name)
kfputils.upload_pipeline(pipeline_yaml_path, _name)
run_name = kfputils.generate_run_name(_name)
kfputils.run_pipeline(
run_name=run_name,
experiment_name=self.pipeline.config.experiment_name,
pipeline_package_path=pipeline_yaml_path)
def compile_notebook(request, source_notebook_path,
notebook_metadata_overrides=None, debug=False):
"""Compile the notebook to KFP DSL."""
instance = Kale(source_notebook_path, notebook_metadata_overrides, debug)
instance.logger = request.log if hasattr(request, "log") else logger
pipeline_graph, pipeline_parameters = instance.notebook_to_graph()
script_path = instance.generate_kfp_executable(pipeline_graph,
pipeline_parameters)
pipeline_name = instance.pipeline_metadata["pipeline_name"]
package_path = kfputils.compile_pipeline(script_path, pipeline_name)
return {"pipeline_package_path": os.path.relpath(package_path),
"pipeline_metadata": instance.pipeline_metadata}
def compile_notebook(request, source_notebook_path,
notebook_metadata_overrides=None, debug=False):
"""Compile the notebook to KFP DSL."""
processor = NotebookProcessor(source_notebook_path,
notebook_metadata_overrides)
pipeline = processor.to_pipeline()
script_path = Compiler(pipeline).compile()
# FIXME: Why were we tapping into the Kale logger?
# instance = Kale(source_notebook_path, notebook_metadata_overrides, debug)
# instance.logger = request.log if hasattr(request, "log") else logger
package_path = kfputils.compile_pipeline(script_path,
pipeline.config.pipeline_name)
return {"pipeline_package_path": os.path.relpath(package_path),
"pipeline_metadata": pipeline.config.to_dict()}
def main():
"""Entry-point of CLI command."""
parser = argparse.ArgumentParser(description=ARGS_DESC,
formatter_class=RawTextHelpFormatter)
general_group = parser.add_argument_group('General')
general_group.add_argument('--nb',
type=str,
help='Path to source JupyterNotebook',
required=True)
# use store_const instead of store_true because we None instead of
# False in case the flag is missing
general_group.add_argument('--upload_pipeline',
action='store_const',
const=True)
general_group.add_argument('--run_pipeline',
action='store_const',
const=True)
general_group.add_argument('--debug', action='store_true')
metadata_group = parser.add_argument_group('Notebook Metadata Overrides',
METADATA_GROUP_DESC)
metadata_group.add_argument('--experiment_name',
type=str,
help='Name of the created experiment')
metadata_group.add_argument('--pipeline_name',
type=str,
help='Name of the deployed pipeline')
metadata_group.add_argument('--pipeline_description',
type=str,
help='Description of the deployed pipeline')
metadata_group.add_argument('--docker_image',
type=str,
help='Docker base image used to build the '
'pipeline steps')
metadata_group.add_argument('--kfp_host',
type=str,
help='KFP endpoint. Provide address as '
'<host>:<port>.')
metadata_group.add_argument('--storage-class-name',
type=str,
help='The storage class name for the created'
' volumes')
metadata_group.add_argument('--volume-access-mode',
type=str,
help='The access mode for the created volumes')
args = parser.parse_args()
# get the notebook metadata args group
mt_overrides_group = next(
filter(lambda x: x.title == 'Notebook Metadata Overrides',
parser._action_groups))
# get the single args of that group
mt_overrides_group_dict = {
a.dest: getattr(args, a.dest, None)
for a in mt_overrides_group._group_actions
if getattr(args, a.dest, None) is not None
}
# FIXME: We are removing the `debug` arg. This shouldn't be an issue
processor = NotebookProcessor(args.nb, mt_overrides_group_dict)
pipeline = processor.run()
dsl_script_path = Compiler(pipeline).compile()
pipeline_name = pipeline.config.pipeline_name
pipeline_package_path = kfputils.compile_pipeline(dsl_script_path,
pipeline_name)
if args.upload_pipeline:
kfputils.upload_pipeline(pipeline_package_path=pipeline_package_path,
pipeline_name=pipeline_name,
host=pipeline.config.kfp_host)
if args.run_pipeline:
run_name = kfputils.generate_run_name(pipeline_name)
kfputils.run_pipeline(run_name=run_name,
experiment_name=pipeline.config.experiment_name,
pipeline_package_path=pipeline_package_path,
host=pipeline.config.kfp_host)
def main():
"""Entry-point of CLI command."""
parser = argparse.ArgumentParser(description=ARGS_DESC,
formatter_class=RawTextHelpFormatter)
general_group = parser.add_argument_group('General')
general_group.add_argument('--nb',
type=str,
help='Path to source JupyterNotebook',
required=True)
# use store_const instead of store_true because we None instead of
# False in case the flag is missing
general_group.add_argument('--upload_pipeline',
action='store_const',
const=True)
general_group.add_argument('--run_pipeline',
action='store_const',
const=True)
general_group.add_argument('--debug', action='store_true')
metadata_group = parser.add_argument_group('Notebook Metadata Overrides',
METADATA_GROUP_DESC)
metadata_group.add_argument('--experiment_name',
type=str,
help='Name of the created experiment')
metadata_group.add_argument('--pipeline_name',
type=str,
help='Name of the deployed pipeline')
metadata_group.add_argument('--pipeline_description',
type=str,
help='Description of the deployed pipeline')
metadata_group.add_argument('--docker_image',
type=str,
help='Docker base image used to build the '
'pipeline steps')
metadata_group.add_argument('--kfp_host',
type=str,
help='KFP endpoint. Provide address as '
'<host>:<port>.')
args = parser.parse_args()
# get the notebook metadata args group
mt_overrides_group = next(
filter(lambda x: x.title == 'Notebook Metadata Overrides',
parser._action_groups))
# get the single args of that group
mt_overrides_group_dict = {
a.dest: getattr(args, a.dest, None)
for a in mt_overrides_group._group_actions
if getattr(args, a.dest, None) is not None
}
kale = Kale(source_notebook_path=args.nb,
notebook_metadata_overrides=mt_overrides_group_dict,
debug=args.debug)
pipeline_graph, pipeline_parameters = kale.notebook_to_graph()
script_path = kale.generate_kfp_executable(pipeline_graph,
pipeline_parameters)
# compile the pipeline to kfp tar package
pipeline_name = kale.pipeline_metadata['pipeline_name']
pipeline_package_path = kfputils.compile_pipeline(script_path,
pipeline_name)
if args.upload_pipeline:
kfputils.upload_pipeline(
pipeline_package_path=pipeline_package_path,
pipeline_name=kale.pipeline_metadata['pipeline_name'],
host=kale.pipeline_metadata.get('kfp_host', None))
if args.run_pipeline:
run_name = kfputils.generate_run_name(
kale.pipeline_metadata['pipeline_name'])
kfputils.run_pipeline(
run_name=run_name,
experiment_name=kale.pipeline_metadata['experiment_name'],
pipeline_package_path=pipeline_package_path,
host=kale.pipeline_metadata.get('kfp_host', None))
