def test_generate_function(config_mock, step_name, source, ins, outs, metadata,
target):
"""Test that python code is generated correctly."""
pipeline = Pipeline(NotebookConfig(**{**DUMMY_NB_CONFIG, **metadata}))
step = Step(name=step_name, source=source, ins=ins, outs=outs)
compiler = Compiler(pipeline)
res = compiler.generate_lightweight_component(step)
target = open(os.path.join(THIS_DIR, "../assets/functions", target)).read()
assert res.strip() == target.strip()
def test_notebook_to_dsl(random_string, notebook_path, dsl_path):
"""Test code generation end to end from notebook to DSL."""
random_string.return_value = "rnd"
overrides = {"abs_working_dir": "/kale"}
pipeline = NotebookProcessor(notebook_path, overrides).to_pipeline()
dsl_script_path = Compiler(pipeline).compile()
expected_result = open(dsl_path).read()
result = open(dsl_script_path).read()
assert result == expected_result
def _do(*args, **do_kwargs):
if not utils.main_source_lives_in_cwd():
# XXX: See arrikto/dev#671 for more details
raise RuntimeError(
"Kale does not yet support running a pipeline when"
" Python's current working directory is different from the"
" location of the source script. You are now running"
" `python %s`. Consider moving into the source script"
" directory with `cd %s` and running `python %s`,"
" instead.\nPlease reach out to the Arrikto team in case"
" you need more information and assistance." %
(sys.argv[0], os.path.dirname(
sys.argv[0]), os.path.basename(sys.argv[0])))
if args:
raise RuntimeError("Positional arguments found in pipeline"
" function call `%s`. Please provide just"
" keyword arguments." % func.__name__)
cli_args = _parse_cli_args()
if cli_args.kfp and not cli_args.dry_run:
volumes = rokutils.interactive_snapshot_and_get_volumes()
kwargs["volumes"] = volumes
config = PipelineConfig(**kwargs)
processor = PythonProcessor(func, config)
pipeline_obj = processor.run()
pipeline_obj.override_pipeline_parameters_from_kwargs(**do_kwargs)
if cli_args.kfp:
if cli_args.dry_run:
return Compiler(pipeline_obj).compile()
else:
return Compiler(pipeline_obj).compile_and_run()
else: # run the pipeline locally
return pipeline_obj.run()
def compile_notebook(request, source_notebook_path,
notebook_metadata_overrides=None, debug=False):
"""Compile the notebook to KFP DSL."""
processor = NotebookProcessor(source_notebook_path,
notebook_metadata_overrides)
pipeline = processor.to_pipeline()
script_path = Compiler(pipeline).compile()
# FIXME: Why were we tapping into the Kale logger?
# instance = Kale(source_notebook_path, notebook_metadata_overrides, debug)
# instance.logger = request.log if hasattr(request, "log") else logger
package_path = kfputils.compile_pipeline(script_path,
pipeline.config.pipeline_name)
return {"pipeline_package_path": os.path.relpath(package_path),
"pipeline_metadata": pipeline.config.to_dict()}
def main():
"""Entry-point of CLI command."""
parser = argparse.ArgumentParser(description=ARGS_DESC,
formatter_class=RawTextHelpFormatter)
general_group = parser.add_argument_group('General')
general_group.add_argument('--nb',
type=str,
help='Path to source JupyterNotebook',
required=True)
# use store_const instead of store_true because we None instead of
# False in case the flag is missing
general_group.add_argument('--upload_pipeline',
action='store_const',
const=True)
general_group.add_argument('--run_pipeline',
action='store_const',
const=True)
general_group.add_argument('--debug', action='store_true')
metadata_group = parser.add_argument_group('Notebook Metadata Overrides',
METADATA_GROUP_DESC)
metadata_group.add_argument('--experiment_name',
type=str,
help='Name of the created experiment')
metadata_group.add_argument('--pipeline_name',
type=str,
help='Name of the deployed pipeline')
metadata_group.add_argument('--pipeline_description',
type=str,
help='Description of the deployed pipeline')
metadata_group.add_argument('--docker_image',
type=str,
help='Docker base image used to build the '
'pipeline steps')
metadata_group.add_argument('--kfp_host',
type=str,
help='KFP endpoint. Provide address as '
'<host>:<port>.')
metadata_group.add_argument('--storage-class-name',
type=str,
help='The storage class name for the created'
' volumes')
metadata_group.add_argument('--volume-access-mode',
type=str,
help='The access mode for the created volumes')
args = parser.parse_args()
# get the notebook metadata args group
mt_overrides_group = next(
filter(lambda x: x.title == 'Notebook Metadata Overrides',
parser._action_groups))
# get the single args of that group
mt_overrides_group_dict = {
a.dest: getattr(args, a.dest, None)
for a in mt_overrides_group._group_actions
if getattr(args, a.dest, None) is not None
}
# FIXME: We are removing the `debug` arg. This shouldn't be an issue
processor = NotebookProcessor(args.nb, mt_overrides_group_dict)
pipeline = processor.run()
dsl_script_path = Compiler(pipeline).compile()
pipeline_name = pipeline.config.pipeline_name
pipeline_package_path = kfputils.compile_pipeline(dsl_script_path,
pipeline_name)
if args.upload_pipeline:
kfputils.upload_pipeline(pipeline_package_path=pipeline_package_path,
pipeline_name=pipeline_name,
host=pipeline.config.kfp_host)
if args.run_pipeline:
run_name = kfputils.generate_run_name(pipeline_name)
kfputils.run_pipeline(run_name=run_name,
experiment_name=pipeline.config.experiment_name,
pipeline_package_path=pipeline_package_path,
host=pipeline.config.kfp_host)