def test_chunk_dict():
assert chunk_dict({}) == []
d = {"a": 1, "b": 2, "c": 3}
assert chunk_dict(d) == [{"a": 1}, {"b": 2}, {"c": 3}]
assert chunk_dict(d, 2) == [{"a": 1, "b": 2}, {"c": 3}]
assert chunk_dict(d, 3) == [d]
assert chunk_dict(d, 4) == [d]
def create_stage_from_cli(repo: "Repo",
single_stage: bool = False,
fname: str = None,
**kwargs: Any) -> Union["Stage", "PipelineStage"]:
from dvc.dvcfile import PIPELINE_FILE
from . import PipelineStage, Stage, create_stage, restore_meta
if single_stage:
kwargs.pop("name", None)
stage_cls = Stage
path = fname or _get_file_path(kwargs)
else:
stage_name = kwargs.get("name", None)
path = PIPELINE_FILE
stage_cls = PipelineStage
if not (stage_name and is_valid_name(stage_name)):
raise InvalidStageName
params = chunk_dict(parse_params(kwargs.pop("params", [])))
stage = create_stage(stage_cls,
repo=repo,
path=path,
params=params,
**kwargs)
restore_meta(stage)
return stage
def create_stage_from_cli(
repo: "Repo",
single_stage: bool = False,
fname: str = None,
validate: bool = False,
force: bool = False,
**kwargs: Any,
) -> Union["Stage", "PipelineStage"]:
from dvc.dvcfile import PIPELINE_FILE
from . import PipelineStage, Stage, create_stage, restore_meta
cmd = kwargs.get("cmd")
if not cmd:
raise InvalidArgumentError("command is not specified")
stage_name = kwargs.get("name")
if stage_name and single_stage:
raise InvalidArgumentError(
"`-n|--name` is incompatible with `--single-stage`")
if stage_name and fname:
raise InvalidArgumentError(
"`--file` is currently incompatible with `-n|--name` "
"and requires `--single-stage`")
if not stage_name and not single_stage:
raise InvalidArgumentError("`-n|--name` is required")
if single_stage:
kwargs.pop("name", None)
stage_cls = Stage
path = fname or _get_file_path(kwargs)
else:
path = PIPELINE_FILE
stage_cls = PipelineStage
if not (stage_name and is_valid_name(stage_name)):
raise InvalidStageName
kwargs["cmd"] = cmd[0] if isinstance(cmd, list) and len(cmd) == 1 else cmd
kwargs["live_summary"] = not kwargs.pop("live_no_summary", False)
kwargs["live_html"] = not kwargs.pop("live_no_html", False)
params = chunk_dict(parse_params(kwargs.pop("params", [])))
stage = create_stage(stage_cls,
repo=repo,
path=path,
params=params,
**kwargs)
if validate:
validate_state(repo, stage, force=force)
restore_meta(stage)
return stage
def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.exceptions import InvalidArgumentError, OutputDuplicationError
from dvc.stage import PipelineStage, Stage, create_stage, restore_meta
from dvc.stage.exceptions import InvalidStageName
if not kwargs.get("cmd"):
raise InvalidArgumentError("command is not specified")
stage_cls = PipelineStage
path = PIPELINE_FILE
stage_name = kwargs.get("name")
if stage_name and single_stage:
raise InvalidArgumentError(
"`-n|--name` is incompatible with `--single-stage`")
if stage_name and fname:
raise InvalidArgumentError(
"`--file` is currently incompatible with `-n|--name` "
"and requires `--single-stage`")
if not stage_name and not single_stage:
raise InvalidArgumentError("`-n|--name` is required")
if single_stage:
kwargs.pop("name", None)
stage_cls = Stage
path = fname or _get_file_path(kwargs)
else:
if not is_valid_name(stage_name):
raise InvalidStageName
params = chunk_dict(parse_params_from_cli(kwargs.pop("params", [])))
stage = create_stage(stage_cls,
repo=self,
path=path,
params=params,
**kwargs)
restore_meta(stage)
if kwargs.get("run_cache", True) and stage.can_be_skipped:
return None
dvcfile = Dvcfile(self, stage.path)
try:
if kwargs.get("force", True):
with suppress(ValueError):
self.stages.remove(stage)
else:
_check_stage_exists(dvcfile, stage)
self.check_modified_graph([stage])
except OutputDuplicationError as exc:
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
if no_exec:
stage.ignore_outs()
else:
stage.run(
no_commit=kwargs.get("no_commit", False),
run_cache=kwargs.get("run_cache", True),
)
dvcfile.dump(stage, update_lock=not no_exec)
return stage