def run(runner, cfg_path, commands, arg, splits):
"""Run commands within pipelines using runner named RUNNER."""
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg_module = importlib.import_module(cfg_path)
args = dict(arg)
args = convert_bool_args(args)
cfgs = get_configs(cfg_module, runner, args)
runner = registry.get_runner(runner)()
for cfg in cfgs:
cfg.update()
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.recursive_validate_config()
cfg_dict = cfg.dict()
cfg_json_uri = cfg.get_config_uri()
json_to_file(cfg_dict, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def run(runner: str, cfg_module: str, commands: List[str],
arg: List[Tuple[str, str]], splits: int):
"""Run COMMANDS within pipelines in CFG_MODULE using RUNNER.
RUNNER: name of the Runner to use
CFG_MODULE: the module with `get_configs` function that returns PipelineConfigs.
This can either be a Python module path or a local path to a .py file.
COMMANDS: space separated sequence of commands to run within pipeline. The order in
which to run them is based on the Pipeline.commands attribute. If this is omitted,
all commands will be run.
"""
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
args = dict(arg)
args = convert_bool_args(args)
cfgs = get_configs(cfg_module, runner, args)
runner = registry.get_runner(runner)()
for cfg in cfgs:
cfg.update()
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.recursive_validate_config()
cfg_dict = cfg.dict()
cfg_json_uri = cfg.get_config_uri()
json_to_file(cfg_dict, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def run(runner: str, cfg_module: str, commands: List[str],
arg: List[Tuple[str, str]], splits: int):
"""Subcommand to run commands within pipelines using runner named RUNNER.
Args:
runner: name of runner to use
cfg_module: name of module with `get_configs` function that returns
PipelineConfigs
commands: names of commands to run within pipeline. The order in which
to run them is based on the Pipeline.commands attribute. If this is
omitted, all commands will be run.
"""
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg_module = importlib.import_module(cfg_module)
args = dict(arg)
args = convert_bool_args(args)
cfgs = get_configs(cfg_module, runner, args)
runner = registry.get_runner(runner)()
for cfg in cfgs:
cfg.update()
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.recursive_validate_config()
cfg_dict = cfg.dict()
cfg_json_uri = cfg.get_config_uri()
json_to_file(cfg_dict, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def save_pipeline_config(cfg: 'PipelineConfig', output_uri: str):
"""Save a PipelineConfig to JSON file.
Inject rv_config and plugin_versions before saving.
"""
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.plugin_versions = registry.plugin_versions
cfg_json = cfg.json()
str_to_file(cfg_json, output_uri)
def _run_pipeline(cfg, runner, tmp_dir, splits=1, commands=None):
cfg.update()
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.recursive_validate_config()
# This is to run the validation again to check any fields that may have changed
# after the Config was constructed, possibly by the update method.
build_config(cfg.dict())
cfg_json = cfg.json()
cfg_json_uri = cfg.get_config_uri()
str_to_file(cfg_json, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)