def test(self):
# NOTE: using 'copy' so that cache and link don't have same inode
ret = main(["config", "cache.type", "copy"])
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.assertEqual(len(stages[0].outs), 1)
out = stages[0].outs[0]
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
_, entry_hash = next(
self.dvc.cache.local.load_dir_cache(out.hash_info).items())
cache = os.fspath(
self.dvc.cache.local.tree.hash_to_path_info(entry_hash.value))
os.chmod(cache, 0o644)
with open(cache, "w+") as fobj:
fobj.write("1")
with pytest.raises(CheckoutError):
self.dvc.checkout(force=True)
self.assertFalse(os.path.exists(cache))
def test(self):
"""
Making sure that 'remote' syntax is handled properly for local outs.
"""
cwd = os.getcwd()
remote = "myremote"
ret = main(["remote", "add", remote, cwd])
self.assertEqual(ret, 0)
self.dvc = DvcRepo()
foo = f"remote://{remote}/{self.FOO}"
ret = main(["add", foo])
self.assertEqual(ret, 0)
d = load_yaml("foo.dvc")
self.assertEqual(d["outs"][0]["path"], foo)
bar = os.path.join(cwd, self.BAR)
ret = main(["add", bar])
self.assertEqual(ret, 0)
d = load_yaml("bar.dvc")
self.assertEqual(d["outs"][0]["path"], self.BAR)
def test(self):
# Import
with StaticFileServer() as httpd:
import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp_url(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
self.dvc.remove("imported_file.dvc")
with StaticFileServer(handler_class=ContentMD5Handler) as httpd:
import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp_url(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
# Run --deps
with StaticFileServer() as httpd:
remote = self.get_remote(httpd.server_port)
cache_id = str(uuid.uuid4())
cache = urljoin(remote, cache_id)
ret1 = main(["remote", "add", "mycache", cache])
ret2 = main(["remote", "add", "myremote", remote])
self.assertEqual(ret1, 0)
self.assertEqual(ret2, 0)
self.dvc = DvcRepo(".")
run_dependency = urljoin(remote, self.BAR)
run_output = "remote_file"
cmd = 'open("{}", "w+")'.format(run_output)
with open("create-output.py", "w") as fd:
fd.write(cmd)
run_stage = self.dvc.run(
deps=[run_dependency],
outs=[run_output],
cmd="python create-output.py",
)
self.assertTrue(run_stage is not None)
self.assertTrue(os.path.exists(run_output))
# Pull
self.dvc.remove(import_stage.path, outs_only=True)
self.assertFalse(os.path.exists(import_output))
shutil.move(self.local_cache, cache_id)
self.assertFalse(os.path.exists(self.local_cache))
self.dvc.pull([import_stage.path], remote="mycache")
self.assertTrue(os.path.exists(import_output))
def setUp(self):
super().setUp()
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
self.dvc.close()
self.dvc = DvcRepo(".")
def test_ssh_dir_out(dvc_repo):
if not _should_test_ssh():
pytest.skip()
# Set up remote and cache
remote_url = get_ssh_url()
assert main(["remote", "add", "upstream", remote_url]) == 0
cache_url = get_ssh_url()
assert main(["remote", "add", "sshcache", cache_url]) == 0
assert main(["config", "cache.ssh", "sshcache"]) == 0
# Recreating to reread configs
repo = DvcRepo(dvc_repo.root_dir)
url_info = URLInfo(remote_url)
mkdir_cmd = "mkdir dir-out;cd dir-out;echo 1 > 1.txt; echo 2 > 2.txt"
repo.run(
cmd="ssh {netloc} 'cd {path};{cmd}'".format(netloc=url_info.netloc,
path=url_info.path,
cmd=mkdir_cmd),
outs=[(url_info / "dir-out").url],
deps=["foo"], # add a fake dep to not consider this a callback
)
repo.reproduce("dir-out.dvc")
repo.reproduce("dir-out.dvc", force=True)
def test(self):
# NOTE: using 'copy' so that cache and link don't have same inode
ret = main(["config", "cache.type", "copy"])
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
stages = self.dvc.add(self.DATA_DIR)
self.assertEqual(len(stages), 1)
self.assertEqual(len(stages[0].outs), 1)
out = stages[0].outs[0]
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
cache = self.dvc.cache.local.get(checksum)
os.chmod(cache, 0o644)
with open(cache, "w+") as fobj:
fobj.write("1")
with pytest.raises(CheckoutError):
self.dvc.checkout(force=True)
self.assertFalse(os.path.exists(cache))
def setUp(self):
super(
TestShouldNotCheckoutUponCorruptedLocalHardlinkCache, self
).setUp()
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
def test(self):
"""
Making sure that 'remote' syntax is handled properly for local outs.
"""
cwd = os.getcwd()
remote = "myremote"
ret = main(["remote", "add", remote, cwd])
self.assertEqual(ret, 0)
self.dvc = DvcRepo()
foo = "remote://{}/{}".format(remote, self.FOO)
ret = main(["add", foo])
self.assertEqual(ret, 0)
with open("foo.dvc", "r") as fobj:
d = yaml.safe_load(fobj)
self.assertEqual(d["outs"][0]["path"], foo)
bar = os.path.join(cwd, self.BAR)
ret = main(["add", bar])
self.assertEqual(ret, 0)
with open("bar.dvc", "r") as fobj:
d = yaml.safe_load(fobj)
self.assertEqual(d["outs"][0]["path"], bar)
def _test_metrics(self, func):
self.dvc.scm.commit("init")
self.dvc.scm.branch("one")
self.dvc.scm.branch("two")
func("master")
func("one")
func("two")
# TestDvc currently is based on TestGit, so it is safe to use
# scm.git for now
self.dvc.scm.repo.git.clean("-fd")
self.dvc = DvcRepo(".")
res = self.dvc.metrics.show(["metrics.json"],
all_branches=True,
typ="json",
xpath="metrics")
self.assertEqual(
res,
{
"master": {
"metrics.json": ["master"]
},
"one": {
"metrics.json": ["one"]
},
"two": {
"metrics.json": ["two"]
},
"working tree": {
"metrics.json": ["two"]
},
},
)
res = self.dvc.metrics.show(all_branches=True,
typ="json",
xpath="metrics")
self.assertEqual(
res,
{
"master": {
"metrics.json": ["master"]
},
"one": {
"metrics.json": ["one"]
},
"two": {
"metrics.json": ["two"]
},
"working tree": {
"metrics.json": ["two"]
},
},
)
def test(self):
from dvc.repo import Repo as DvcRepo
self.dvc = DvcRepo(self._root_dir)
with self.assertRaises(MoveNotDataSourceError):
self.dvc.move(self.file1, "dst")
ret = main(["move", self.file1, "dst"])
self.assertNotEqual(ret, 0)
def test_allow_init_dvc_subdir(tmp_dir, scm, monkeypatch):
tmp_dir.gen({"subdir": {}})
with monkeypatch.context() as m:
m.chdir("subdir")
assert main(["init", "--subdir"]) == 0
repo = DvcRepo("subdir")
assert repo.root_dir == os.fspath(tmp_dir / "subdir")
assert repo.scm.root_dir == os.fspath(tmp_dir)
def _test_metrics(self, func):
self.dvc.scm.commit("init")
self.dvc.scm.branch("one")
self.dvc.scm.branch("two")
func("master")
func("one")
func("two")
self.dvc = DvcRepo(".")
res = self.dvc.metrics.show("metrics.json",
all_branches=True,
typ="json",
xpath="metrics")
self.assertEqual(
res,
{
"master": {
"metrics.json": ["master"]
},
"one": {
"metrics.json": ["one"]
},
"two": {
"metrics.json": ["two"]
},
},
)
res = self.dvc.metrics.show("",
all_branches=True,
typ="json",
xpath="metrics")
self.assertEqual(
res,
{
"master": {
"metrics.json": ["master"]
},
"one": {
"metrics.json": ["one"]
},
"two": {
"metrics.json": ["two"]
},
},
)
def setUp(self):
super().setUp()
self.additional_path = TestDir.mkdtemp()
self.additional_git = Repo.init(self.additional_path)
self.additional_dvc = DvcRepo.init(self.additional_path)
cache_path = os.path.join(self._root_dir, ".dvc", "cache")
config_path = os.path.join(self.additional_path, ".dvc",
"config.local")
cfg = configobj.ConfigObj()
cfg.filename = config_path
cfg["cache"] = {"dir": cache_path}
cfg.write()
self.additional_dvc = DvcRepo(self.additional_path)
def test_ssh_dir_out(tmp_dir, dvc, ssh_server):
tmp_dir.gen({"foo": "foo content"})
# Set up remote and cache
user = ssh_server.test_creds["username"]
port = ssh_server.port
keyfile = ssh_server.test_creds["key_filename"]
remote_url = SSHMocked.get_url(user, port)
assert main(["remote", "add", "upstream", remote_url]) == 0
assert main(["remote", "modify", "upstream", "keyfile", keyfile]) == 0
cache_url = SSHMocked.get_url(user, port)
assert main(["remote", "add", "sshcache", cache_url]) == 0
assert main(["config", "cache.ssh", "sshcache"]) == 0
assert main(["remote", "modify", "sshcache", "keyfile", keyfile]) == 0
# Recreating to reread configs
repo = DvcRepo(dvc.root_dir)
# To avoid "WARNING: UNPROTECTED PRIVATE KEY FILE" from ssh
os.chmod(keyfile, 0o600)
(tmp_dir / "script.py").write_text(
"import sys, pathlib\n"
"path = pathlib.Path(sys.argv[1])\n"
"dir_out = path / 'dir-out'\n"
"dir_out.mkdir()\n"
"(dir_out / '1.txt').write_text('1')\n"
"(dir_out / '2.txt').write_text('2')\n"
)
url_info = URLInfo(remote_url)
repo.run(
cmd="python {} {}".format(tmp_dir / "script.py", url_info.path),
outs=["remote://upstream/dir-out"],
deps=["foo"], # add a fake dep to not consider this a callback
)
repo.reproduce("dir-out.dvc")
repo.reproduce("dir-out.dvc", force=True)
def test(self, mock_prompt):
if not self.should_test():
return
cache = (self.scheme + self.scheme_sep + self.bucket + self.sep +
str(uuid.uuid4()))
ret = main(["config", "cache." + self.cache_scheme, "myrepo"])
self.assertEqual(ret, 0)
ret = main(["remote", "add", "myrepo", cache])
self.assertEqual(ret, 0)
ret = main(["remote", "modify", "myrepo", "type", self.cache_type])
self.assertEqual(ret, 0)
remote_name = "myremote"
remote_key = str(uuid.uuid4())
remote = (self.scheme + self.scheme_sep + self.bucket + self.sep +
remote_key)
ret = main(["remote", "add", remote_name, remote])
self.assertEqual(ret, 0)
ret = main(["remote", "modify", remote_name, "type", self.cache_type])
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
foo_key = remote_key + self.sep + self.FOO
bar_key = remote_key + self.sep + self.BAR
foo_path = (self.scheme + self.scheme_sep + self.bucket + self.sep +
foo_key)
bar_path = (self.scheme + self.scheme_sep + self.bucket + self.sep +
bar_key)
# Using both plain and remote notation
out_foo_path = "remote://" + remote_name + "/" + self.FOO
out_bar_path = bar_path
self.write(self.bucket, foo_key, self.FOO_CONTENTS)
import_stage = self.dvc.imp_url(out_foo_path, "import")
self.assertTrue(os.path.exists("import"))
self.assertTrue(filecmp.cmp("import", self.FOO, shallow=False))
self.assertEqual(self.dvc.status([import_stage.path]), {})
self.check_already_cached(import_stage)
import_remote_stage = self.dvc.imp_url(out_foo_path,
out_foo_path + "_imported")
self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
cmd_stage = self.dvc.run(
outs=[out_bar_path],
deps=[out_foo_path],
cmd=self.cmd(foo_path, bar_path),
)
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
self.assertEqual(self.dvc.status(), {})
self.check_already_cached(cmd_stage)
self.write(self.bucket, foo_key, self.BAR_CONTENTS)
self.assertNotEqual(self.dvc.status(), {})
self.dvc.update(import_stage.path)
self.assertTrue(os.path.exists("import"))
self.assertTrue(filecmp.cmp("import", self.BAR, shallow=False))
self.assertEqual(self.dvc.status([import_stage.path]), {})
self.dvc.update(import_remote_stage.path)
self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
stages = self.dvc.reproduce(cmd_stage.path)
self.assertEqual(len(stages), 1)
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
self.assertEqual(self.dvc.status(), {})
self.dvc.gc()
self.assertEqual(self.dvc.status(), {})
self.dvc.remove(cmd_stage.path, outs_only=True)
self.assertNotEqual(self.dvc.status([cmd_stage.path]), {})
self.dvc.checkout([cmd_stage.path], force=True)
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
def setUp(self):
super().setUp()
self.tmpdir = TestDvc.mkdtemp()
ret = main(["config", "cache.type", "hardlink"])
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
