def test_params_order(tmp_dir, dvc):
tmp_dir.gen(
{
"params.yaml": dumps_yaml({"p": 1}),
"params1.yaml": dumps_yaml({"p1": 1}),
"sub": {"params2.yaml": dumps_yaml({"p2": 1})},
}
)
params_path = os.path.join("..", "params.yaml")
p2_path = os.path.join("sub", "params2.yaml")
dvc.stage.add(
params=[{p2_path: ["p2"]}, {"params1.yaml": ["p1"]}],
cmd="cmd1",
name="stage1",
)
with (tmp_dir / "sub").chdir():
dvc.stage.add(params=[{params_path: ["p"]}], cmd="cmd2", name="stage2")
# params are sorted during dumping, therefore p1 is first
assert list(dvc.params.show()[""]) == [
"params1.yaml",
p2_path,
"params.yaml",
]
def test_params_order(tmp_dir, dvc, dummy_stage):
tmp_dir.gen(
{
"params.yaml": dumps_yaml({"p1": 1}),
"p1.yaml": dumps_yaml({"p2": 1}),
"sub": {"p2.yaml": dumps_yaml({"p3": 1})},
}
)
p2_path = os.path.join("sub", "p2.yaml")
sub_stage = os.path.join("sub", "dvc.yaml")
dummy_stage(params=[{p2_path: ["p3"]}, {"p1.yaml": ["p2"]}])
dummy_stage(path=sub_stage, params=["p1"])
# params are sorted during dumping, therefore p1 is first
assert list(dvc.params.show()[""]) == ["p1.yaml", p2_path, "params.yaml"]
def test_parametrization_is_not_enabled_by_default(tmp_dir, dvc, mocker):
assert dvc.config["feature"]["parametrization"] is False
(tmp_dir / "dvc.yaml").write_text(dumps_yaml(RESOLVED_DVC_YAML_DATA))
mock = mocker.patch("dvc.dvcfile.DataResolver")
stages = list(Dvcfile(dvc, "dvc.yaml").stages)
mock.assert_not_called()
assert len(stages) == 2
def test_load_metric_from_dict_yaml(tmp_dir):
metric = [{"acccuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
dmetric = {"train": metric}
plot_data = YAMLPlotData("-", "revision", dumps_yaml(dmetric))
expected = metric
for d in expected:
d["rev"] = "revision"
assert list(map(dict, plot_data.to_datapoints())) == expected
def test_load_vars_with_relpath(tmp_dir, scm, dvc):
tmp_dir.scm_gen(DEFAULT_PARAMS_FILE, dumps_yaml(DATA), commit="add params")
subdir = tmp_dir / "subdir"
d = {"vars": [os.path.relpath(tmp_dir / DEFAULT_PARAMS_FILE, subdir)]}
revisions = ["HEAD", "workspace"]
for rev in dvc.brancher(revs=["HEAD"]):
assert rev == revisions.pop()
resolver = DataResolver(dvc, subdir, d)
assert resolver.context == deepcopy(DATA)
def test_load_from(mocker):
d = {"x": {"y": {"z": 5}, "lst": [1, 2, 3]}, "foo": "foo"}
fs = mocker.Mock(
open=mock_open(read_data=dumps_yaml(d)),
**{
"exists.return_value": True,
"isdir.return_value": False
},
)
file = "params.yaml"
c = Context.load_from(fs, file)
assert asdict(c["x"].meta) == {
"source": file,
"dpaths": ["x"],
"local": False,
}
assert asdict(c["foo"].meta) == {
"source": file,
"local": False,
"dpaths": ["foo"],
}
assert asdict(c["x"]["y"].meta) == {
"source": file,
"dpaths": ["x", "y"],
"local": False,
}
assert asdict(c["x"]["y"]["z"].meta) == {
"source": file,
"dpaths": ["x", "y", "z"],
"local": False,
}
assert asdict(c["x"]["lst"].meta) == {
"source": file,
"dpaths": ["x", "lst"],
"local": False,
}
assert asdict(c["x"]["lst"][0].meta) == {
"source": file,
"dpaths": ["x", "lst", "0"],
"local": False,
}
def init(
repo: "Repo",
name: str = None,
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
stream: Optional[TextIO] = None,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
name = name or type
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults.copy() if defaults else {}
overrides = overrides.copy() if overrides else {}
with_live = type == "live"
if interactive:
defaults = init_interactive(
name,
validator=validate_prompts,
defaults=defaults,
live=with_live,
provided=overrides,
stream=stream,
)
else:
if with_live:
# suppress `metrics`/`plots` if live is selected, unless
# it is also provided via overrides/cli.
# This makes output to be a checkpoint as well.
defaults.pop("metrics", None)
defaults.pop("plots", None)
else:
defaults.pop("live", None) # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params_kv = []
params = context.get("params")
if params:
params_kv.append(loadd_params(params))
checkpoint_out = bool(context.get("live"))
models = context.get("models")
stage = repo.stage.create(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=params_kv,
metrics_no_cache=compact([context.get("metrics")]),
plots_no_cache=compact([context.get("plots")]),
live=context.get("live"),
force=force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
if interactive:
ui.error_write(Rule(style="green"), styled=True)
_yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
syn = Syntax(_yaml, "yaml", theme="ansi_dark")
ui.error_write(syn, styled=True)
from dvc.ui.prompt import Confirm
if not interactive or Confirm.ask(
"Do you want to add the above contents to dvc.yaml?",
console=ui.error_console,
default=True,
stream=stream,
):
with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
stage.dump(update_lock=False)
stage.ignore_outs()
if params:
repo.scm_context.track_file(params)
else:
raise DvcException("Aborting ...")
return stage
import pytest
from dvc.dvcfile import PIPELINE_LOCK, Lockfile
from dvc.stage.utils import split_params_deps
from dvc.utils.fs import remove
from dvc.utils.serialize import dumps_yaml, parse_yaml_for_update
from dvc.utils.strictyaml import YAMLValidationError, make_relpath
from dvc_data.hashfile.hash_info import HashInfo
from tests.func.test_run_multistage import supported_params
FS_STRUCTURE = {
"foo": "bar\nfoobar",
"bar": "foo\nfoobar",
"foobar": "foobar\nbar",
"params.yaml": dumps_yaml(supported_params),
"params2.yaml": dumps_yaml(supported_params),
}
def read_lock_file(file=PIPELINE_LOCK):
with open(file, encoding="utf-8") as f:
data = parse_yaml_for_update(f.read(), file)
assert isinstance(data, OrderedDict)
return data
def assert_eq_lockfile(previous, new):
for content in (previous, new):
assert isinstance(content, OrderedDict)
def is_serializable(d):
json.dumps(d)
dumps_yaml(d)
return True
def init(
repo: "Repo",
name: str = None,
type: str = "default", # pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
dvcfile = make_dvcfile(repo, "dvc.yaml")
name = name or type
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults or {}
overrides = overrides or {}
with_live = type == "live"
if interactive:
defaults = init_interactive(
name,
defaults=defaults,
live=with_live,
provided=overrides,
show_tree=True,
)
else:
if with_live:
# suppress `metrics`/`params` if live is selected, unless
# it is also provided via overrides/cli.
# This makes output to be a checkpoint as well.
defaults.pop("metrics")
defaults.pop("params")
else:
defaults.pop("live") # suppress live otherwise
context: Dict[str, str] = {**defaults, **overrides}
assert "cmd" in context
params_kv = []
if context.get("params"):
from dvc.utils.serialize import LOADERS
path = context["params"]
assert isinstance(path, str)
_, ext = os.path.splitext(path)
params_kv = [{path: list(LOADERS[ext](path))}]
checkpoint_out = bool(context.get("live"))
models = context.get("models")
stage = repo.stage.create(
name=name,
cmd=context["cmd"],
deps=compact([context.get("code"),
context.get("data")]),
params=params_kv,
metrics_no_cache=compact([context.get("metrics")]),
plots_no_cache=compact([context.get("plots")]),
live=context.get("live"),
force=force,
**{"checkpoints" if checkpoint_out else "outs": compact([models])},
)
if interactive:
ui.write(Rule(style="green"), styled=True)
_yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
syn = Syntax(_yaml, "yaml", theme="ansi_dark")
ui.error_write(syn, styled=True)
if not interactive or ui.confirm(
"Do you want to add the above contents to dvc.yaml?"):
with _disable_logging():
stage.dump(update_lock=False)
stage.ignore_outs()
else:
raise DvcException("Aborting ...")
return stage
def resolve(self):
stages = self.data.get(STAGES_KWD, {})
data = join(starmap(self._resolve_entry, stages.items()))
logger.trace("Resolved dvc.yaml:\n%s", dumps_yaml(data))
return {STAGES_KWD: data}
