def _run_command(cfg_json_uri, command, split_ind=None, num_splits=None, runner=None): pipeline_cfg_dict = file_to_json(cfg_json_uri) rv_config_dict = pipeline_cfg_dict.get('rv_config') rv_config.reset( config_overrides=rv_config_dict, verbosity=rv_config.verbosity, tmp_dir=rv_config.tmp_dir) tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name cfg = build_config(pipeline_cfg_dict) pipeline = cfg.build(tmp_dir) if num_splits is not None and split_ind is None and runner is not None: runner = registry.get_runner(runner)() split_ind = runner.get_split_ind() command_fn = getattr(pipeline, command) if num_splits is not None and num_splits > 1: msg = 'Running {} command split {}/{}...'.format( command, split_ind + 1, num_splits) click.echo(click.style(msg, fg='green')) command_fn(split_ind=split_ind, num_splits=num_splits) else: msg = 'Running {} command...'.format(command) click.echo(click.style(msg, fg='green')) command_fn()
def run(runner, cfg_path, commands, arg, splits): """Run commands within pipelines using runner named RUNNER.""" tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name cfg_module = importlib.import_module(cfg_path) args = dict(arg) args = convert_bool_args(args) cfgs = get_configs(cfg_module, runner, args) runner = registry.get_runner(runner)() for cfg in cfgs: cfg.update() cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema) cfg.recursive_validate_config() cfg_dict = cfg.dict() cfg_json_uri = cfg.get_config_uri() json_to_file(cfg_dict, cfg_json_uri) pipeline = cfg.build(tmp_dir) if not commands: commands = pipeline.commands runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def run(runner: str, cfg_module: str, commands: List[str], arg: List[Tuple[str, str]], splits: int): """Subcommand to run commands within pipelines using runner named RUNNER. Args: runner: name of runner to use cfg_module: name of module with `get_configs` function that returns PipelineConfigs commands: names of commands to run within pipeline. The order in which to run them is based on the Pipeline.commands attribute. If this is omitted, all commands will be run. """ tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name cfg_module = importlib.import_module(cfg_module) args = dict(arg) args = convert_bool_args(args) cfgs = get_configs(cfg_module, runner, args) runner = registry.get_runner(runner)() for cfg in cfgs: cfg.update() cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema) cfg.recursive_validate_config() cfg_dict = cfg.dict() cfg_json_uri = cfg.get_config_uri() json_to_file(cfg_dict, cfg_json_uri) pipeline = cfg.build(tmp_dir) if not commands: commands = pipeline.commands runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def run(runner: str, cfg_module: str, commands: List[str], arg: List[Tuple[str, str]], splits: int): """Run COMMANDS within pipelines in CFG_MODULE using RUNNER. RUNNER: name of the Runner to use CFG_MODULE: the module with `get_configs` function that returns PipelineConfigs. This can either be a Python module path or a local path to a .py file. COMMANDS: space separated sequence of commands to run within pipeline. The order in which to run them is based on the Pipeline.commands attribute. If this is omitted, all commands will be run. """ tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name args = dict(arg) args = convert_bool_args(args) cfgs = get_configs(cfg_module, runner, args) runner = registry.get_runner(runner)() for cfg in cfgs: cfg.update() cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema) cfg.recursive_validate_config() cfg_dict = cfg.dict() cfg_json_uri = cfg.get_config_uri() json_to_file(cfg_dict, cfg_json_uri) pipeline = cfg.build(tmp_dir) if not commands: commands = pipeline.commands runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def _run_command(cfg_json_uri: str, command: str, split_ind: Optional[int] = None, num_splits: Optional[int] = None, runner: Optional[str] = None): """Run a single command using a serialized PipelineConfig. Args: cfg_json_uri: URI of a JSON file with a serialized PipelineConfig command: name of command to run split_ind: the index that a split command should assume num_splits: the total number of splits to use runner: the name of the runner to use """ pipeline_cfg_dict = file_to_json(cfg_json_uri) rv_config_dict = pipeline_cfg_dict.get('rv_config') rv_config.set_everett_config(profile=rv_config.profile, config_overrides=rv_config_dict) tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name cfg = build_config(pipeline_cfg_dict) pipeline = cfg.build(tmp_dir) if num_splits is not None and split_ind is None and runner is not None: runner = registry.get_runner(runner)() split_ind = runner.get_split_ind() command_fn = getattr(pipeline, command) if num_splits is not None and num_splits > 1: msg = 'Running {} command split {}/{}...'.format( command, split_ind + 1, num_splits) click.echo(click.style(msg, fg='green')) command_fn(split_ind=split_ind, num_splits=num_splits) else: msg = 'Running {} command...'.format(command) click.echo(click.style(msg, fg='green')) command_fn()
def run(runner: str, cfg_module: str, commands: List[str], arg: List[Tuple[str, str]], splits: int): """Run COMMANDS within pipelines in CFG_MODULE using RUNNER. RUNNER: name of the Runner to use CFG_MODULE: the module with `get_configs` function that returns PipelineConfigs. This can either be a Python module path or a local path to a .py file. COMMANDS: space separated sequence of commands to run within pipeline. The order in which to run them is based on the Pipeline.commands attribute. If this is omitted, all commands will be run. """ tmp_dir_obj = rv_config.get_tmp_dir() tmp_dir = tmp_dir_obj.name args = dict(arg) args = convert_bool_args(args) cfgs = get_configs(cfg_module, runner, args) runner = registry.get_runner(runner)() for cfg in cfgs: _run_pipeline(cfg, runner, tmp_dir, splits, commands)