def test_read_params_py(tmp_dir, dvc):
parameters_file = "parameters.py"
tmp_dir.gen(
parameters_file,
"INT: int = 5\n"
"FLOAT = 0.001\n"
"STR = 'abc'\n"
"BOOL: bool = True\n"
"DICT = {'a': 1}\n"
"LIST = [1, 2, 3]\n"
"SET = {4, 5, 6}\n"
"TUPLE = (10, 100)\n"
"NONE = None\n",
)
dep = ParamsDependency(
Stage(dvc),
parameters_file,
[
"INT",
"FLOAT",
"STR",
"BOOL",
"DICT",
"LIST",
"SET",
"TUPLE",
"NONE",
],
)
assert dep.read_params() == {
"INT": 5,
"FLOAT": 0.001,
"STR": "abc",
"BOOL": True,
"DICT": {
"a": 1
},
"LIST": [1, 2, 3],
"SET": {4, 5, 6},
"TUPLE": (10, 100),
"NONE": None,
}
tmp_dir.gen(parameters_file,
"class Train:\n foo = 'val1'\n bar = 'val2'\n")
dep = ParamsDependency(Stage(dvc), parameters_file, ["Train.foo"])
assert dep.read_params() == {"Train.foo": "val1"}
dep = ParamsDependency(Stage(dvc), parameters_file, ["Train"])
assert dep.read_params() == {"Train": {"foo": "val1", "bar": "val2"}}
tmp_dir.gen(
parameters_file,
"x = 4\n"
"config.x = 3\n"
"class Klass:\n"
" def __init__(self):\n"
" self.a = 'val1'\n"
" container.a = 2\n"
" self.container.a = 1\n"
" a = 'val2'\n",
)
dep = ParamsDependency(Stage(dvc), parameters_file, ["x", "Klass.a"])
assert dep.read_params() == {"x": 4, "Klass.a": "val1"}
def _get_output(self):
stage = Stage(self.dvc)
return LocalOutput(stage, "path")
def test_meta_ignored():
stage = Stage(None, "path", cmd="mycmd")
d = dict(TEST_STAGE_DICT, meta={"author": "Suor"})
with mock.patch.object(stage, "dumpd", return_value=d):
assert stage.compute_md5() == "e9521a22111493406ea64a88cda63e0b"
def test_always_changed(dvc):
stage = Stage(dvc, "path", always_changed=True)
stage.save()
with dvc.lock:
assert stage.changed()
assert stage.status()["path"] == ["always changed"]
def _get(self, path):
return _get(Stage(self.dvc), path, None)
def test_wdir_default_ignored():
stage = Stage(None, "path", cmd="mycmd")
d = dict(TEST_STAGE_DICT, wdir=".")
with mock.patch.object(stage, "dumpd", return_value=d):
assert stage.compute_md5() == "e9521a22111493406ea64a88cda63e0b"
def test_read_params_nonexistent_file(dvc):
dep = ParamsDependency(Stage(dvc), None, ["foo"])
assert dep.read_params() == {}
def test_load_from_pipeline_illegal_type(dvc, key):
stage = Stage(dvc)
with pytest.raises(ValueError):
output.load_from_pipeline(stage, [key], "outs")
with pytest.raises(ValueError):
output.load_from_pipeline(stage, [{"key": key}], "outs")
def test_dumpd_with_info(dvc):
dep = ParamsDependency(Stage(dvc), None, PARAMS)
assert dep.dumpd() == {"path": DEFAULT_PARAMS_FILE, "params": PARAMS}
def test_dumpd_without_info(dvc):
dep = ParamsDependency(Stage(dvc), None, list(PARAMS.keys()))
assert dep.dumpd() == {
"path": DEFAULT_PARAMS_FILE,
"params": list(PARAMS.keys()),
}
def test_params_error(dvc, params):
with pytest.raises(ValueError):
loads_params(Stage(dvc), params)
def test_get_hash_missing_param(tmp_dir, dvc):
tmp_dir.gen(DEFAULT_PARAMS_FILE, "bar: baz")
dep = ParamsDependency(Stage(dvc), None, ["foo"])
with pytest.raises(MissingParamsError):
dep.get_hash()
def test_get_hash_missing_config(dvc):
dep = ParamsDependency(Stage(dvc), None, ["foo"])
with pytest.raises(MissingParamsError):
dep.get_hash()
def _get_out(dvc, path):
return _get(Stage(dvc), path, None, None, None, None)
def test_read_params_unsupported_format(tmp_dir, dvc):
tmp_dir.gen(DEFAULT_PARAMS_FILE, b"\0\1\2\3\4\5\6\7")
dep = ParamsDependency(Stage(dvc), None, ["foo"])
with pytest.raises(BadParamFileError):
dep.read_params()
def test_load_from_pipeline_error_on_typ(dvc, typ):
with pytest.raises(ValueError):
output.load_from_pipeline(Stage(dvc), ["file1"], typ)
def _get_output(self):
stage = Stage(self.dvc)
return OutputLOCAL(stage, "path")
def test_save_missing(dvc, mocker):
stage = Stage(dvc)
dep = Dependency(stage, "path")
with mocker.patch.object(dep.fs, "exists", return_value=False):
with pytest.raises(dep.DoesNotExistError):
dep.save()
def test_always_changed():
stage = Stage(None, "path", always_changed=True)
stage.save()
assert stage.changed()
assert stage.status()["path"] == ["always changed"]
def test_stage_checksum():
stage = Stage(None, "path", cmd="mycmd")
with mock.patch.object(stage, "dumpd", return_value=TEST_STAGE_DICT):
assert stage.compute_md5() == "e9521a22111493406ea64a88cda63e0b"
def move(self, from_path, to_path):
"""
Renames an output file and modifies the stage associated
to reflect the change on the pipeline.
If the output has the same name as its stage, it would
also rename the corresponding .dvc file.
E.g.
Having: (hello, hello.dvc)
$ dvc move hello greetings
Result: (greeting, greeting.dvc)
It only works with outputs generated by `add` or `import`,
also known as data sources.
"""
import dvc.output as Output
from dvc.stage import Stage
from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile
from_out = Output.loads_from(Stage(self), [from_path])[0]
assert from_out.scheme == "local"
to_path = _expand_target_path(from_path, to_path)
outs = self.find_outs_by_path(from_out.fspath)
assert len(outs) == 1
out = outs[0]
stage = out.stage
if not stage.is_data_source:
raise MoveNotDataSourceError(stage.addressing)
stage_name = os.path.splitext(os.path.basename(stage.path))[0]
from_name = os.path.basename(from_out.fspath)
if stage_name == from_name:
new_fname = os.path.join(
os.path.dirname(to_path),
os.path.basename(to_path) + DVC_FILE_SUFFIX,
)
new_wdir = os.path.abspath(
os.path.join(os.curdir, os.path.dirname(to_path)))
to_path = os.path.relpath(to_path, new_wdir)
new_stage = self.stage.create(
single_stage=True,
fname=new_fname,
wdir=new_wdir,
outs=[to_path],
)
os.unlink(stage.path)
stage = new_stage
else:
to_path = os.path.relpath(to_path, stage.wdir)
to_out = Output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
out.move(to_out)
stage.save()
Dvcfile(self, stage.path).dump(stage)
def test_wdir_non_default_is_not_ignored():
stage = Stage(None, "path", cmd="mycmd")
d = dict(TEST_STAGE_DICT, wdir="..")
with mock.patch.object(stage, "dumpd", return_value=d):
assert stage.compute_md5() == "2ceba15e87f6848aa756502c1e6d24e9"
def _get_output(self):
stage = Stage(self.dvc)
return self._get_cls()(stage, "path", cache=False)
def test(self):
stage = Stage(None, "path")
stage.wdir = os.path.join("..", "..")
self.assertEqual(stage.dumpd()["wdir"], "../..")
def test_str_workdir_outside_repo(erepo_dir):
stage = Stage(erepo_dir.dvc)
output = LocalOutput(stage, "path", cache=False)
assert relpath("path", erepo_dir.dvc.root_dir) == str(output)
def test_read_params_default_loader(tmp_dir, dvc):
parameters_file = "parameters.foo"
dump_yaml(parameters_file, {"some": {"path": {"foo": ["val1", "val2"]}}})
dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
def test_read_params_wrong_suffix(tmp_dir, dvc):
parameters_file = "parameters.toml"
dump_yaml(parameters_file, {"some": {"path": {"foo": ["val1", "val2"]}}})
dep = ParamsDependency(Stage(dvc), parameters_file, ["some.path.foo"])
with pytest.raises(BadParamFileError):
dep.read_params()
