def test_get_from_non_dvc_master(
tmp_dir, erepo_dir, tmp_path, monkeypatch, caplog
):
with monkeypatch.context() as m:
m.chdir(fspath(erepo_dir))
erepo_dir.scm.checkout("new_branch", create_new=True)
erepo_dir.scm_gen(
{"some_file": "some_contents"}, commit="create some file"
)
erepo_dir.scm.checkout("master")
erepo_dir.dvc.scm.repo.index.remove([".dvc"], r=True)
erepo_dir.dvc.scm.commit("remove .dvc")
# sanity check
with pytest.raises(UrlNotDvcRepoError):
Repo.get(fspath(erepo_dir), "some_file")
caplog.clear()
dst = "file_imported"
with caplog.at_level(logging.INFO, logger="dvc"):
Repo.get(fspath(erepo_dir), "some_file", out=dst, rev="new_branch")
assert caplog.text == ""
assert (tmp_dir / dst).read_text() == "some_contents"
def test_get_repo_dir(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
Repo.get(os.fspath(erepo_dir), "dir", "dir_imported")
assert (tmp_dir / "dir_imported").read_text() == {"file": "contents"}
def test_get_from_non_dvc_repo(tmp_dir, erepo_dir):
erepo_dir.scm.repo.index.remove([erepo_dir.dvc.dvc_dir], r=True)
erepo_dir.scm.commit("remove dvc")
erepo_dir.scm_gen({"some_file": "contents"}, commit="create file")
Repo.get(fspath(erepo_dir), "some_file", "file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_from_non_dvc_master(tmp_dir, git_dir):
with git_dir.chdir(), git_dir.branch("branch", new=True):
git_dir.init(dvc=True)
git_dir.dvc_gen("some_file", "some text", commit="create some file")
Repo.get(os.fspath(git_dir), "some_file", out="some_dst", rev="branch")
assert (tmp_dir / "some_dst").read_text() == "some text"
def test_get_repo_file(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
Repo.get(fspath(erepo_dir), "file", "file_imported")
assert os.path.isfile("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_repo_dir(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
Repo.get(fspath(erepo_dir), "dir", "dir_imported")
assert os.path.isdir("dir_imported")
trees_equal(fspath(erepo_dir / "dir"), "dir_imported")
def test_get_repo_dir(erepo):
src = erepo.DATA_DIR
dst = erepo.DATA_DIR + "_imported"
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isdir(dst)
trees_equal(src, dst)
def test_get_repo_file(erepo):
src = erepo.FOO
dst = erepo.FOO + "_imported"
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isfile(dst)
assert filecmp.cmp(erepo.FOO, dst, shallow=False)
def test_granular_get_from_subrepos(tmp_dir, erepo_dir):
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
with subrepo.chdir():
subrepo.dvc_gen({"dir": {"bar": "bar"}}, commit="files in subrepo")
path = os.path.join("subrepo", "dir", "bar")
Repo.get(os.fspath(erepo_dir), path, out="out")
assert (tmp_dir / "out").read_text() == "bar"
def test_get_git_file(tmp_dir, erepo):
src = "some_file"
dst = "some_file_imported"
erepo.scm_gen({src: "hello"}, commit="add a regular file")
Repo.get(os.fspath(erepo), src, dst)
assert (tmp_dir / dst).read_text() == "hello"
def test_get_repo_dir(tmp_dir, erepo_dir, monkeypatch):
with monkeypatch.context() as m:
m.chdir(fspath(erepo_dir))
erepo_dir.dvc_gen({"dir": {"file": "contents"}}, commit="create dir")
Repo.get(fspath(erepo_dir), "dir", "dir_imported")
assert os.path.isdir("dir_imported")
trees_equal(fspath(erepo_dir / "dir"), "dir_imported")
def test_get_repo_rev(erepo):
src = "version"
dst = src
Repo.get(erepo.root_dir, src, dst, rev="branch")
assert os.path.exists(dst)
assert os.path.isfile(dst)
with open(dst, "r+") as fobj:
assert fobj.read() == "branch"
def test_get_repo_rev(tmp_dir, erepo_dir, monkeypatch):
with monkeypatch.context() as m:
m.chdir(fspath(erepo_dir))
erepo_dir.scm.checkout("new_branch", create_new=True)
erepo_dir.dvc_gen("file", "contents", commit="create file on branch")
erepo_dir.scm.checkout("master")
Repo.get(fspath(erepo_dir), "file", "file_imported", rev="new_branch")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory):
path = tmp_path_factory.mktemp("ext")
external_data = path / "ext_data"
external_data.write_text("ext_data")
with erepo_dir.chdir():
erepo_dir.dvc_add(fspath(external_data), commit="add external data")
Repo.get(fspath(erepo_dir), fspath(external_data), "ext_data_imported")
assert (tmp_dir / "ext_data_imported").read_text() == "ext_data"
def test_get_git_dir(tmp_dir, erepo_dir):
src = "some_directory"
dst = "some_directory_imported"
erepo_dir.scm_gen({src: {"file.txt": "hello"}}, commit="add a regular dir")
Repo.get(fspath(erepo_dir), src, dst)
assert (tmp_dir / dst).is_dir()
trees_equal(fspath(erepo_dir / src), fspath(tmp_dir / dst))
def test_get_to_dir(tmp_dir, erepo_dir, dname):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
makedirs(dname, exist_ok=True)
Repo.get(fspath(erepo_dir), "file", dname)
assert (tmp_dir / dname).is_dir()
assert (tmp_dir / dname / "file").read_text() == "contents"
def test_get_git_dir(request, tmp_dir, repo_type):
erepo_dir = request.getfixturevalue(repo_type)
src = "some_directory"
dst = "some_directory_imported"
erepo_dir.scm_gen({src: {"file.txt": "hello"}}, commit="add a regular dir")
Repo.get(os.fspath(erepo_dir), src, dst)
assert (tmp_dir / dst).is_dir()
trees_equal(erepo_dir / src, tmp_dir / dst)
def test_get_git_dir(tmp_dir, erepo):
src = "some_directory"
dst = "some_directory_imported"
erepo.scm_gen(
{src: {"dir": {"file.txt": "hello"}}}, commit="add a regular dir"
)
Repo.get(os.fspath(erepo), src, dst)
assert (tmp_dir / dst).read_text() == {"dir": {"file.txt": "hello"}}
def test_get_git_file(request, tmp_dir, repo_type):
erepo_dir = request.getfixturevalue(repo_type)
src = "some_file"
dst = "some_file_imported"
erepo_dir.scm_gen({src: "hello"}, commit="add a regular file")
Repo.get(os.fspath(erepo_dir), src, dst)
assert (tmp_dir / dst).is_file()
assert (tmp_dir / dst).read_text() == "hello"
def test_get_from_subrepos(tmp_dir, erepo_dir, is_dvc, files):
subrepo = erepo_dir / "subrepo"
make_subrepo(subrepo, erepo_dir.scm)
gen = subrepo.dvc_gen if is_dvc else subrepo.scm_gen
with subrepo.chdir():
gen(files, commit="add files in subrepo")
key = next(iter(files))
Repo.get(os.fspath(erepo_dir), f"subrepo/{key}", out="out")
assert (tmp_dir / "out").read_text() == files[key]
def test_get_mixed_dir(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen(os.path.join("dir", "foo"), "foo", commit="foo")
erepo_dir.scm_gen(os.path.join("dir", "bar"), "bar", commit="bar")
Repo.get(os.fspath(erepo_dir), "dir")
assert (tmp_dir / "dir").read_text() == {
".gitignore": "/foo\n",
"foo": "foo",
"bar": "bar",
}
def test_get_to_dir(dname, erepo):
src = erepo.FOO
makedirs(dname, exist_ok=True)
Repo.get(erepo.root_dir, src, dname)
dst = os.path.join(dname, os.path.basename(src))
assert os.path.isdir(dname)
assert filecmp.cmp(erepo.FOO, dst, shallow=False)
def test_non_cached_output(tmp_path, erepo):
os.chdir(erepo.root_dir)
erepo.dvc.run(outs_no_cache=["non_cached_file"],
cmd="echo hello > non_cached_file")
erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"])
erepo.dvc.scm.commit("add non-cached output")
os.chdir(fspath_py35(tmp_path))
Repo.get(erepo.root_dir, "non_cached_file")
src = os.path.join(erepo.root_dir, "non_cached_file")
assert os.path.isfile("non_cached_file")
assert filecmp.cmp(src, "non_cached_file", shallow=False)
def test_get_from_non_dvc_master(tmp_dir, git_dir, caplog):
with git_dir.chdir(), git_dir.branch("branch", new=True):
git_dir.init(dvc=True)
git_dir.dvc_gen("some_file", "some text", commit="create some file")
caplog.clear()
with caplog.at_level(logging.INFO, logger="dvc"):
Repo.get(fspath(git_dir), "some_file", out="some_dst", rev="branch")
assert caplog.text == ""
assert (tmp_dir / "some_dst").read_text() == "some text"
def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory, monkeypatch):
path = tmp_path_factory.mktemp("ext")
external_data = path / "ext_data"
external_data.write_text("ext_data")
with monkeypatch.context() as m:
m.chdir(fspath(erepo_dir))
erepo_dir.dvc.add(fspath(external_data))
erepo_dir.scm_add(["ext_data.dvc"], commit="add external data")
Repo.get(fspath(erepo_dir), fspath(external_data), "ext_data_imported")
assert (tmp_dir / "ext_data_imported").is_file()
assert (tmp_dir / "ext_data_imported").read_text() == "ext_data"
def run(self):
try:
Repo.get(
self.args.url,
path=self.args.path,
out=self.args.out,
rev=self.args.rev,
)
return 0
except DvcException:
logger.exception("failed to get '{}' from '{}'".format(
self.args.path, self.args.url))
return 1
def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, local_remote,
run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
dvc.scm.commit("add pipeline stage")
dvc.push()
with git_dir.chdir():
Repo.get("file:///{}".format(fspath(tmp_dir)), "bar", out="baz")
assert (git_dir / "baz").read_text() == "foo"
def test_get_regular_file(erepo):
src = "some_file"
dst = "some_file_imported"
src_path = os.path.join(erepo.root_dir, src)
erepo.create(src_path, "hello")
erepo.dvc.scm.add([src_path])
erepo.dvc.scm.commit("add a regular file")
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isfile(dst)
assert filecmp.cmp(src_path, dst, shallow=False)
def test_get_regular_dir(erepo):
src = "some_directory"
dst = "some_directory_imported"
src_file_path = os.path.join(erepo.root_dir, src, "file.txt")
erepo.create(src_file_path, "hello")
erepo.dvc.scm.add([src_file_path])
erepo.dvc.scm.commit("add a regular dir")
Repo.get(erepo.root_dir, src, dst)
assert os.path.exists(dst)
assert os.path.isdir(dst)
trees_equal(os.path.join(erepo.root_dir, src), dst)
def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc.config.set(Config.SECTION_CACHE,
Config.SECTION_CACHE_TYPE, "symlink")
erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
erepo_dir.scm_add([erepo_dir.dvc.config.config_file],
"set cache type to symlinks")
erepo_dir.dvc_gen("file", "contents", "create file")
assert System.is_symlink(erepo_dir / "file")
Repo.get(fspath(erepo_dir), "file", "file_imported")
assert not System.is_symlink("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
