def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Convert Notebook to python script') \ .add_work_dir_argument() \ .add_conf_path_argument() \ .add_docstring_conf() \ .add_force_argument() \ .add_argument('-n', '--notebook', type=str, required=True, help='The notebook to convert') \ .parse(args) self.set_log_level(args) conf = self.get_conf(args.working_directory, args.notebook, args.conf_path) if not conf.path: raise MlVToolException('Configuration file is mandatory') docstring_conf_path = args.docstring_conf or conf.docstring_conf docstring_conf = load_docstring_conf( docstring_conf_path) if docstring_conf_path else None output_script = get_script_output_path(args.notebook, conf) out_dvc_cmd = get_dvc_cmd_output_path(output_script, conf) self.check_force(args.force, [output_script, out_dvc_cmd]) export_to_script(args.notebook, output_script, conf) gen_dvc_command(output_script, out_dvc_cmd, conf, docstring_conf)
def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Checks all notebooks and scripts consistency.\n' 'Run the up to date checks on all notebooks from the notebook directory. ' 'Script names are deduce from the conf.') \ .add_work_dir_argument() \ .add_conf_path_argument() \ .add_path_argument('-n', '--notebooks-dir', type=str, help='Notebooks directory') \ .add_argument('-i', '--ignore', action='append', help='Notebook filename to ignore', default=[]) \ .parse(args) self.set_log_level(args) conf = self.get_conf(args.working_directory, args.notebooks_dir, args.conf_path) if not conf.path: raise MlVToolException('Configuration file is mandatory') equals = True for notebook in glob.glob(join(args.notebooks_dir, '*.ipynb')): if basename(notebook) in args.ignore: logging.info(f'Ignore notebook {notebook}') continue associated_script = get_script_output_path(notebook, conf) equals = run_consistency_check(notebook, associated_script, conf) and equals sys.exit(0 if equals else 1)
def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Generate python script wrappers') \ .add_work_dir_argument() \ .add_conf_path_argument() \ .add_force_argument() \ .add_docstring_conf() \ .add_path_argument('-i', '--input-script', type=str, required=True, help='The python input script') \ .add_path_argument('-o', '--out-dvc-cmd', type=str, help='Path to the generated bash dvc command') \ .parse(args) self.set_log_level(args) conf = self.get_conf(args.working_directory, args.input_script, args.conf_path) docstring_conf_path = args.docstring_conf or conf.docstring_conf if not conf.path and not args.out_dvc_cmd: raise MlVToolException( 'Parameter --out-dvc-cmd is mandatory if no conf provided') docstring_conf = load_docstring_conf( docstring_conf_path) if docstring_conf_path else None out_dvc_cmd = args.out_dvc_cmd or get_dvc_cmd_output_path( args.input_script, conf) self.check_force(args.force, [out_dvc_cmd]) gen_dvc_command(args.input_script, out_dvc_cmd, conf, docstring_conf)
def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Checks notebook and script consistency') \ .add_work_dir_argument() \ .add_conf_path_argument() \ .add_path_argument('-n', '--notebook', type=str, help='The notebook to check') \ .add_path_argument('-s', '--script', required=True, type=str, help='The script to check') \ .parse(args) self.set_log_level(args) conf = self.get_conf(args.working_directory, args.notebook, args.conf_path) equals = run_consistency_check(args.notebook, args.script, conf) sys.exit(0 if equals else 1)
def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Export a DVC pipeline to sequential execution.') \ .add_force_argument() \ .add_work_dir_argument() \ .add_argument('--dvc', type=str, required=True, help='DVC targeted pipeline metadata step') \ .add_argument('-o', '--output', type=str, help='The Python pipeline script output path', required=True) \ .parse(args) self.set_log_level(args) work_dir = args.working_directory if not args.force and exists(args.output): raise MlVToolException( f'Output file {args.output} already exists, use --force option to overwrite it' ) export_pipeline(args.dvc, args.output, work_dir)
def run(self, *args, **kwargs): args = ArgumentBuilder(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Convert Notebook to python script') \ .add_work_dir_argument() \ .add_conf_path_argument() \ .add_force_argument() \ .add_path_argument('-n', '--notebook', type=str, required=True, help='The notebook to convert') \ .add_path_argument('-o', '--output', type=str, help='The Python script output path') \ .parse(args) self.set_log_level(args) conf = self.get_conf(args.working_directory, args.notebook, args.conf_path) if not conf.path and not args.output: raise MlVToolException( 'Parameter --output is mandatory if no conf provided') output = args.output or get_script_output_path(args.notebook, conf) self.check_force(args.force, [output]) export_to_script(args.notebook, output, conf)