def test_add_static_arguments(): parser = CustomParser() with parser: parser.add_argument('--static-arg', '-s') added = {'static_arg', 's'} assert set(parser.static_args) & added == added
def test_add_dynamic_arguments(): parser = CustomParser() with parser: parser.add_argument('--static-arg', '-s') parser.add_argument('--dynamic-arg', '-d') added = {'dynamic_arg', 'd'} assert not set(parser.static_args) & added
def main(): parser = CustomParser(description='Make a pipeline report') with parser: parser.add_argument( '--output', '-o', help='Where to save the report, defaults to pipeline.html', default='pipeline.html') dag, args = _custom_command(parser) dag.to_markup(path=args.output) print('Report saved at:', args.output)
def main(): parser = CustomParser(description='Plot a pipeline') with parser: parser.add_argument( '--output', '-o', help='Where to save the plot, defaults to pipeline.png', default='pipeline.png') dag, args = _custom_command(parser) dag.plot(output=args.output) print('Plot saved at:', args.output)
def main(): start_time = datetime.datetime.now() parser = CustomParser(description='Make a pipeline report', prog='ploomber report') with parser: parser.add_argument( '--output', '-o', help='Where to save the report, defaults to pipeline.html', default='pipeline.html') dag, args = parser.load_from_entry_point_arg() dag.to_markup(path=args.output) end_time = datetime.datetime.now() telemetry.log_api("ploomber_report", total_runtime=str(end_time - start_time), dag=dag, metadata={'argv': sys.argv}) print('Report saved at:', args.output)
def main(): parser = CustomParser(description='Plot a pipeline') with parser: parser.add_argument( '--output', '-o', help='Where to save the plot, defaults to pipeline.png', default=None) dag, args = _custom_command(parser) if args.output is not None: output = args.output else: name = extract_name(args.entry_point) output = 'pipeline.png' if name is None else f'pipeline.{name}.png' dag.plot(output=output) print('Plot saved at:', output)
def main(render_only=False): parser = CustomParser(description='Build pipeline') with parser: parser.add_argument('--force', '-f', help='Force execution by ignoring status', action='store_true', default=False) parser.add_argument('--skip-upstream', '-su', help='Skip building upstream dependencies. ' 'Only applicable when using --partially', action='store_true', default=False) parser.add_argument( '--partially', '-p', help='Build a pipeline partially until certain task', default=None) parser.add_argument( '--debug', '-d', help='Drop a debugger session if an exception happens', action='store_true', default=False) dag, args = _custom_command(parser) # when using the parallel executor from the CLI, ensure we print progress # to stdout if isinstance(dag.executor, Parallel): dag.executor.print_progress = True if render_only: dag.render() else: if args.partially: report = dag.build_partially(args.partially, force=args.force, debug=args.debug, skip_upstream=args.skip_upstream) else: report = dag.build(force=args.force, debug=args.debug) if report: print(report) return dag
def main(): start_time = datetime.datetime.now() parser = CustomParser(description='Plot a pipeline', prog='ploomber plot') with parser: parser.add_argument( '--output', '-o', help='Where to save the plot, defaults to pipeline.png', default=None) dag, args = parser.load_from_entry_point_arg() if args.output is not None: output = args.output else: name = extract_name(args.entry_point) output = 'pipeline.png' if name is None else f'pipeline.{name}.png' dag.plot(output=output) end_time = datetime.datetime.now() telemetry.log_api("ploomber_plot", total_runtime=str(end_time - start_time), dag=dag, metadata={'argv': sys.argv}) print('Plot saved at:', output)
def test_doesnt_modify_error_if_unknown_reason(): def my_fn(log=True): pass parser = CustomParser() arg = Mock() arg.options_strings = ['a', 'b'] parser.add_argument = Mock(side_effect=ArgumentError(None, 'message')) with parser: pass with pytest.raises(ArgumentError) as excinfo: _add_args_from_callable(parser, my_fn) assert 'message' == str(excinfo.value)
def main(render_only=False): parser = CustomParser(description='Build pipeline') with parser: parser.add_argument('--force', '-f', help='Force execution by ignoring status', action='store_true', default=False) parser.add_argument( '--partially', '-p', help='Build a pipeline partially until certain task', default=None) parser.add_argument( '--debug', '-d', help='Drop a debugger session if an exception happens', action='store_true', default=False) dag, args = _custom_command(parser) if render_only: dag.render() else: if args.partially: report = dag.build_partially(args.partially, force=args.force, debug=args.debug) else: report = dag.build(force=args.force, debug=args.debug) if report: print(report) return dag
def test_cannot_add_arguments_without_context_manager(): parser = CustomParser() with pytest.raises(RuntimeError): parser.add_argument('--static-arg', '-s')
def main(): parser = CustomParser(description='Manage scripts and notebooks', prog='ploomber nb') with parser: # The next options do not require a valid entry point # opening .py files as notebooks in JupyterLab with a single click single_click = parser.add_mutually_exclusive_group() single_click.add_argument( '--single-click', '-S', action='store_true', help=('Override JupyterLab defaults to open ' 'scripts as notebook with a single click')) single_click.add_argument( '--single-click-disable', '-d', action='store_true', help=('Disables opening scripts as notebook with a single ' 'click in JupyterLab')) # install/uninstall hook hook = parser.add_mutually_exclusive_group() hook.add_argument('--install-hook', '-I', action='store_true', help='Install git pre-commit hook') hook.add_argument('--uninstall-hook', '-u', action='store_true', help='Uninstall git pre-commit hook') # The next options require a valid entry point # inject/remove cell cell = parser.add_mutually_exclusive_group() cell.add_argument('--inject', '-i', action='store_true', help='Inject cell to all script/notebook tasks') cell.add_argument( '--remove', '-r', action='store_true', help='Remove injected cell in all script/notebook tasks') # re-format parser.add_argument('--format', '-f', help='Re-format all script/notebook tasks') # pair scripts and nbs parser.add_argument('--pair', '-p', help='Pair scripts with ipynb files') # sync scripts and nbs parser.add_argument('--sync', '-s', action='store_true', help='Sync scripts with ipynb files') loading_error = None # commands that need an entry point to work needs_entry_point = {'format', 'inject', 'remove', 'sync', 'pair'} args_ = parser.parse_args() if any(getattr(args_, arg) for arg in needs_entry_point): try: dag, args = parser.load_from_entry_point_arg() except Exception as e: loading_error = e else: dag.render(show_progress=False) if loading_error: err = ('Could not run nb command: the DAG ' 'failed to load') telemetry.log_api("nb_error", metadata={ 'type': 'dag_load_failed', 'exception': err + f' {loading_error}', 'argv': sys.argv }) raise RuntimeError(err) from loading_error else: dag = None args = args_ # options that do not need a DAG if args.single_click: _py_with_single_click_enable() if args.single_click_disable: _py_with_single_click_disable() if args.install_hook: if not Path('.git').is_dir(): err = ('Expected a .git/ directory in the current working ' 'directory. Run this from the repository root directory.') telemetry.log_api("nb_error", metadata={ 'type': 'no_git_config', 'exception': err, 'argv': sys.argv }) raise NotADirectoryError(err) parent = Path('.git', 'hooks') parent.mkdir(exist_ok=True) # pre-commit: remove injected cells _install_hook(parent / 'pre-commit', pre_commit_hook, args.entry_point) click.echo('Successfully installed pre-commit git hook') # post-commit: inject cells _install_hook(parent / 'post-commit', post_commit_hook, args.entry_point) click.echo('Successfully installed post-commit git hook') if args.uninstall_hook: _delete_hook(Path('.git', 'hooks', 'pre-commit')) _delete_hook(Path('.git', 'hooks', 'post-commit')) # options that need a valid DAG if args.format: new_paths = [ str(p) for p in _call_in_source( dag, 'format', 'Formatted notebooks', dict(fmt=args.format), ) if p is not None ] if len(new_paths): click.echo('Extension changed for the following ' f'tasks: {", ".join(new_paths)}. Update your ' 'pipeline declaration.') if args.inject: _call_in_source( dag, 'save_injected_cell', 'Injected celll', dict(), ) click.secho( 'Finished cell injection. Re-run this command if your ' 'pipeline.yaml changes.', fg='green') if args.remove: _call_in_source( dag, 'remove_injected_cell', 'Removed injected cell', dict(), ) if args.sync: # maybe its more efficient to pass all notebook paths at once? _call_in_source(dag, 'sync', 'Synced notebooks') # can pair give trouble if we're reformatting? if args.pair: _call_in_source( dag, 'pair', 'Paired notebooks', dict(base_path=args.pair), ) click.echo(f'Finished pairing notebooks. Tip: add {args.pair!r} to ' 'your .gitignore to keep your repository clean') telemetry.log_api("ploomber_nb", dag=dag, metadata={'argv': sys.argv})
def main(): parser = CustomParser(description='Get task information') with parser: parser.add_argument('task_name') parser.add_argument('--source', '-s', help='Print task source code', action='store_true') parser.add_argument( '--build', '-b', help='Build task (default if no other option is passed)', action='store_true') parser.add_argument('--force', '-f', help='Force build task (ignore up-to-date status)', action='store_true') parser.add_argument('--status', '-st', help='Get task status', action='store_true') parser.add_argument('--on-finish', '-of', help='Only execute on_finish hook', action='store_true') dag, args = _custom_command(parser) dag.render(force=args.force) task = dag[args.task_name] if args.source: print(task.source) if args.status: print(task.status()) if args.on_finish: task._run_on_finish() # task if build by default, but when --source or --status are passed, # the --build flag is required no_flags = not any((args.build, args.status, args.source, args.on_finish)) if no_flags or args.build: task.build(force=args.force)
def main(render_only=False): start_time = datetime.datetime.now() parser = CustomParser(description='Build pipeline', prog='ploomber build') with parser: parser.add_argument('--force', '-f', help='Force execution by ignoring status', action='store_true', default=False) parser.add_argument('--skip-upstream', '-su', help='Skip building upstream dependencies. ' 'Only applicable when using --partially', action='store_true', default=False) parser.add_argument( '--partially', '-p', help='Build a pipeline partially until certain task', default=None) parser.add_argument( '--debug', '-d', help='Drop a debugger session if an exception happens', action='store_true', default=False) # users may try to run "ploomber build {name}" to build a single task if len(sys.argv) > 1 and not sys.argv[1].startswith('-'): suggestion = 'ploomber task {task-name}' cmd_name = parser.prog telemetry.log_api("unsupported_build_cmd", metadata={ 'cmd_name': cmd_name, 'suggestion': suggestion, 'argv': sys.argv }) parser.error(f'{cmd_name!r} does not take positional arguments.\n' f'To build a single task, try: {suggestion!r}') dag, args = parser.load_from_entry_point_arg() # when using the parallel executor from the CLI, ensure we print progress # to stdout if isinstance(dag.executor, Parallel): dag.executor.print_progress = True if render_only: dag.render() else: if args.partially: report = dag.build_partially(args.partially, force=args.force, debug=args.debug, skip_upstream=args.skip_upstream) else: report = dag.build(force=args.force, debug=args.debug) if report: print(report) end_time = datetime.datetime.now() telemetry.log_api("ploomber_build", total_runtime=str(end_time - start_time), dag=dag, metadata={'argv': sys.argv}) return dag
def main(): start_time = datetime.datetime.now() parser = CustomParser(description='Build tasks', prog='ploomber task') with parser: parser.add_argument('task_name') parser.add_argument('--source', '-s', help='Print task source code', action='store_true') parser.add_argument( '--build', '-b', help='Build task (default if no other option is passed)', action='store_true') parser.add_argument('--force', '-f', help='Force task build (ignore up-to-date status)', action='store_true') parser.add_argument('--status', '-st', help='Get task status', action='store_true') parser.add_argument('--on-finish', '-of', help='Only execute on_finish hook', action='store_true') dag, args = parser.load_from_entry_point_arg() dag.render() task = dag[args.task_name] if args.source: print(task.source) if args.status: print(task.status()) if args.on_finish: task._run_on_finish() # task if build by default, but when --source or --status are passed, # the --build flag is required no_flags = not any((args.build, args.status, args.source, args.on_finish)) if no_flags or args.build: task.build(force=args.force) end_time = datetime.datetime.now() telemetry.log_api("ploomber_task", total_runtime=str(end_time - start_time), dag=dag, metadata={'argv': sys.argv})