def test_params_order(tmp_dir, dvc): tmp_dir.gen( { "params.yaml": dumps_yaml({"p": 1}), "params1.yaml": dumps_yaml({"p1": 1}), "sub": {"params2.yaml": dumps_yaml({"p2": 1})}, } ) params_path = os.path.join("..", "params.yaml") p2_path = os.path.join("sub", "params2.yaml") dvc.stage.add( params=[{p2_path: ["p2"]}, {"params1.yaml": ["p1"]}], cmd="cmd1", name="stage1", ) with (tmp_dir / "sub").chdir(): dvc.stage.add(params=[{params_path: ["p"]}], cmd="cmd2", name="stage2") # params are sorted during dumping, therefore p1 is first assert list(dvc.params.show()[""]) == [ "params1.yaml", p2_path, "params.yaml", ]
def test_params_order(tmp_dir, dvc, dummy_stage): tmp_dir.gen( { "params.yaml": dumps_yaml({"p1": 1}), "p1.yaml": dumps_yaml({"p2": 1}), "sub": {"p2.yaml": dumps_yaml({"p3": 1})}, } ) p2_path = os.path.join("sub", "p2.yaml") sub_stage = os.path.join("sub", "dvc.yaml") dummy_stage(params=[{p2_path: ["p3"]}, {"p1.yaml": ["p2"]}]) dummy_stage(path=sub_stage, params=["p1"]) # params are sorted during dumping, therefore p1 is first assert list(dvc.params.show()[""]) == ["p1.yaml", p2_path, "params.yaml"]
def test_parametrization_is_not_enabled_by_default(tmp_dir, dvc, mocker): assert dvc.config["feature"]["parametrization"] is False (tmp_dir / "dvc.yaml").write_text(dumps_yaml(RESOLVED_DVC_YAML_DATA)) mock = mocker.patch("dvc.dvcfile.DataResolver") stages = list(Dvcfile(dvc, "dvc.yaml").stages) mock.assert_not_called() assert len(stages) == 2
def test_load_metric_from_dict_yaml(tmp_dir): metric = [{"acccuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}] dmetric = {"train": metric} plot_data = YAMLPlotData("-", "revision", dumps_yaml(dmetric)) expected = metric for d in expected: d["rev"] = "revision" assert list(map(dict, plot_data.to_datapoints())) == expected
def test_load_vars_with_relpath(tmp_dir, scm, dvc): tmp_dir.scm_gen(DEFAULT_PARAMS_FILE, dumps_yaml(DATA), commit="add params") subdir = tmp_dir / "subdir" d = {"vars": [os.path.relpath(tmp_dir / DEFAULT_PARAMS_FILE, subdir)]} revisions = ["HEAD", "workspace"] for rev in dvc.brancher(revs=["HEAD"]): assert rev == revisions.pop() resolver = DataResolver(dvc, subdir, d) assert resolver.context == deepcopy(DATA)
def test_load_from(mocker): d = {"x": {"y": {"z": 5}, "lst": [1, 2, 3]}, "foo": "foo"} fs = mocker.Mock( open=mock_open(read_data=dumps_yaml(d)), **{ "exists.return_value": True, "isdir.return_value": False }, ) file = "params.yaml" c = Context.load_from(fs, file) assert asdict(c["x"].meta) == { "source": file, "dpaths": ["x"], "local": False, } assert asdict(c["foo"].meta) == { "source": file, "local": False, "dpaths": ["foo"], } assert asdict(c["x"]["y"].meta) == { "source": file, "dpaths": ["x", "y"], "local": False, } assert asdict(c["x"]["y"]["z"].meta) == { "source": file, "dpaths": ["x", "y", "z"], "local": False, } assert asdict(c["x"]["lst"].meta) == { "source": file, "dpaths": ["x", "lst"], "local": False, } assert asdict(c["x"]["lst"][0].meta) == { "source": file, "dpaths": ["x", "lst", "0"], "local": False, }
def init( repo: "Repo", name: str = None, type: str = "default", # pylint: disable=redefined-builtin defaults: Dict[str, str] = None, overrides: Dict[str, str] = None, interactive: bool = False, force: bool = False, stream: Optional[TextIO] = None, ) -> "Stage": from dvc.dvcfile import make_dvcfile dvcfile = make_dvcfile(repo, "dvc.yaml") name = name or type _check_stage_exists(dvcfile, name, force=force) defaults = defaults.copy() if defaults else {} overrides = overrides.copy() if overrides else {} with_live = type == "live" if interactive: defaults = init_interactive( name, validator=validate_prompts, defaults=defaults, live=with_live, provided=overrides, stream=stream, ) else: if with_live: # suppress `metrics`/`plots` if live is selected, unless # it is also provided via overrides/cli. # This makes output to be a checkpoint as well. defaults.pop("metrics", None) defaults.pop("plots", None) else: defaults.pop("live", None) # suppress live otherwise context: Dict[str, str] = {**defaults, **overrides} assert "cmd" in context params_kv = [] params = context.get("params") if params: params_kv.append(loadd_params(params)) checkpoint_out = bool(context.get("live")) models = context.get("models") stage = repo.stage.create( name=name, cmd=context["cmd"], deps=compact([context.get("code"), context.get("data")]), params=params_kv, metrics_no_cache=compact([context.get("metrics")]), plots_no_cache=compact([context.get("plots")]), live=context.get("live"), force=force, **{"checkpoints" if checkpoint_out else "outs": compact([models])}, ) if interactive: ui.error_write(Rule(style="green"), styled=True) _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage))) syn = Syntax(_yaml, "yaml", theme="ansi_dark") ui.error_write(syn, styled=True) from dvc.ui.prompt import Confirm if not interactive or Confirm.ask( "Do you want to add the above contents to dvc.yaml?", console=ui.error_console, default=True, stream=stream, ): with _disable_logging(), repo.scm_context(autostage=True, quiet=True): stage.dump(update_lock=False) stage.ignore_outs() if params: repo.scm_context.track_file(params) else: raise DvcException("Aborting ...") return stage
import pytest from dvc.dvcfile import PIPELINE_LOCK, Lockfile from dvc.stage.utils import split_params_deps from dvc.utils.fs import remove from dvc.utils.serialize import dumps_yaml, parse_yaml_for_update from dvc.utils.strictyaml import YAMLValidationError, make_relpath from dvc_data.hashfile.hash_info import HashInfo from tests.func.test_run_multistage import supported_params FS_STRUCTURE = { "foo": "bar\nfoobar", "bar": "foo\nfoobar", "foobar": "foobar\nbar", "params.yaml": dumps_yaml(supported_params), "params2.yaml": dumps_yaml(supported_params), } def read_lock_file(file=PIPELINE_LOCK): with open(file, encoding="utf-8") as f: data = parse_yaml_for_update(f.read(), file) assert isinstance(data, OrderedDict) return data def assert_eq_lockfile(previous, new): for content in (previous, new): assert isinstance(content, OrderedDict)
def is_serializable(d): json.dumps(d) dumps_yaml(d) return True
def init( repo: "Repo", name: str = None, type: str = "default", # pylint: disable=redefined-builtin defaults: Dict[str, str] = None, overrides: Dict[str, str] = None, interactive: bool = False, force: bool = False, ) -> "Stage": from dvc.dvcfile import make_dvcfile dvcfile = make_dvcfile(repo, "dvc.yaml") name = name or type _check_stage_exists(dvcfile, name, force=force) defaults = defaults or {} overrides = overrides or {} with_live = type == "live" if interactive: defaults = init_interactive( name, defaults=defaults, live=with_live, provided=overrides, show_tree=True, ) else: if with_live: # suppress `metrics`/`params` if live is selected, unless # it is also provided via overrides/cli. # This makes output to be a checkpoint as well. defaults.pop("metrics") defaults.pop("params") else: defaults.pop("live") # suppress live otherwise context: Dict[str, str] = {**defaults, **overrides} assert "cmd" in context params_kv = [] if context.get("params"): from dvc.utils.serialize import LOADERS path = context["params"] assert isinstance(path, str) _, ext = os.path.splitext(path) params_kv = [{path: list(LOADERS[ext](path))}] checkpoint_out = bool(context.get("live")) models = context.get("models") stage = repo.stage.create( name=name, cmd=context["cmd"], deps=compact([context.get("code"), context.get("data")]), params=params_kv, metrics_no_cache=compact([context.get("metrics")]), plots_no_cache=compact([context.get("plots")]), live=context.get("live"), force=force, **{"checkpoints" if checkpoint_out else "outs": compact([models])}, ) if interactive: ui.write(Rule(style="green"), styled=True) _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage))) syn = Syntax(_yaml, "yaml", theme="ansi_dark") ui.error_write(syn, styled=True) if not interactive or ui.confirm( "Do you want to add the above contents to dvc.yaml?"): with _disable_logging(): stage.dump(update_lock=False) stage.ignore_outs() else: raise DvcException("Aborting ...") return stage
def resolve(self): stages = self.data.get(STAGES_KWD, {}) data = join(starmap(self._resolve_entry, stages.items())) logger.trace("Resolved dvc.yaml:\n%s", dumps_yaml(data)) return {STAGES_KWD: data}