def move(self, from_info, to_info):
if (isinstance(from_info, PathInfo)
and from_info.scheme != "local") or (isinstance(
to_info, PathInfo) and to_info.scheme != "local"):
raise NotImplementedError
self.makedirs(to_info.parent)
move(from_info, to_info)
def test_move(tmp_dir):
tmp_dir.gen({"foo": "foo content"})
src = "foo"
dest = os.path.join("some", "directory")
os.makedirs(dest)
assert len(os.listdir(dest)) == 0
move(src, dest)
assert not os.path.isfile(src)
assert len(os.listdir(dest)) == 1
def move(self, from_info, to_info):
if from_info.scheme != "local" or to_info.scheme != "local":
raise NotImplementedError
self.makedirs(to_info.parent)
if self.isfile(from_info):
mode = self._file_mode
else:
mode = self._dir_mode
move(from_info, to_info, mode=mode)
def _download_file(self, from_info, to_info, name, no_progress_bar):
makedirs(to_info.parent, exist_ok=True)
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
name = name or to_info.name
tmp_file = tmp_fname(to_info)
self._download( # noqa, pylint: disable=no-member
from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
)
move(tmp_file, to_info)
def _download_file(self, from_info, to_info, name, no_progress_bar,
file_mode, dir_mode):
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
name = name or to_info.name
tmp_file = tmp_fname(to_info)
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
move(tmp_file, to_info, mode=file_mode)
def test_move(repo_dir):
src = repo_dir.FOO
src_info = PathInfo(repo_dir.BAR)
dest = os.path.join("some", "directory")
dest_info = PathInfo(os.path.join("some", "path-like", "directory"))
os.makedirs(dest)
assert len(os.listdir(dest)) == 0
move(src, dest)
assert not os.path.isfile(src)
assert len(os.listdir(dest)) == 1
os.makedirs(dest_info.fspath)
assert len(os.listdir(dest_info.fspath)) == 0
move(src_info, dest_info)
assert not os.path.isfile(src_info.fspath)
assert len(os.listdir(dest_info.fspath)) == 1
def _pull_cached(self, out, path_info, dest):
with self.state:
tmp = PathInfo(tmp_fname(dest))
src = tmp / path_info.relative_to(out.path_info)
out.path_info = tmp
# Only pull unless all needed cache is present
if out.changed_cache(filter_info=src):
self.cloud.pull(out.get_used_cache(filter_info=src))
failed = out.checkout(filter_info=src)
move(src, dest)
remove(tmp)
if failed:
raise FileNotFoundError
def test_move(tmp_dir):
tmp_dir.gen({"foo": "foo content", "bar": "bar content"})
src = "foo"
src_info = PathInfo("bar")
dest = os.path.join("some", "directory")
dest_info = PathInfo(os.path.join("some", "path-like", "directory"))
os.makedirs(dest)
assert len(os.listdir(dest)) == 0
move(src, dest)
assert not os.path.isfile(src)
assert len(os.listdir(dest)) == 1
os.makedirs(dest_info)
assert len(os.listdir(dest_info)) == 0
move(src_info, dest_info)
assert not os.path.isfile(src_info)
assert len(os.listdir(dest_info)) == 1
def download(
self,
from_info,
to_info,
name=None,
no_progress_bar=False,
file_mode=None,
dir_mode=None,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
return 0
def _download_file(self, from_info, to_info, name, no_progress_bar,
file_mode, dir_mode):
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception as e:
return self._handle_transfer_exception(from_info, to_info, e,
"download")
move(tmp_file, to_info, mode=file_mode)
return 0
def _download_file(
self,
from_info,
to_info,
callback=DEFAULT_CALLBACK,
):
makedirs(to_info.parent, exist_ok=True)
tmp_file = tmp_fname(to_info)
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
try:
# noqa, pylint: disable=no-member
self.get_file(from_info, tmp_file, callback=callback)
except Exception: # pylint: disable=broad-except
# do we need to rollback makedirs for previously not-existing
# directories?
with contextlib.suppress(FileNotFoundError):
os.unlink(tmp_file)
raise
move(tmp_file, to_info)
def _download_file(
self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
):
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
tmp_file = tmp_fname(to_info)
try:
self._download(
from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
return 0
def rename(self, from_path, to_path):
from dvc.utils.fs import move
move(self._path(from_path), self._path(to_path))
def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
(stage, ) = tmp_dir.dvc_gen("foo", "foo")
out = stage.outs[0]
cache = out.cache_path
md5 = out.hash_info.value
foo_objs = out.get_used_objs().get(None, set())
(stage_dir, ) = tmp_dir.dvc_gen({
"data_dir": {
"data_sub_dir": {
"data_sub": "data_sub"
},
"data": "data",
"empty": "",
}
})
out_dir = stage_dir.outs[0]
cache_dir = out_dir.cache_path
md5_dir = out_dir.hash_info.value
dir_objs = {out_dir.obj}
entry_md5s = [entry_obj.hash_info.value for _, entry_obj in out_dir.obj]
def _make_dir_status(status):
expected = {md5_dir: {"name": md5_dir, "status": status}}
expected.update({
entry_md5: {
"name": entry_md5,
"status": status
}
for entry_md5 in entry_md5s
})
return expected
# Check status
status = dvc.cloud.status(foo_objs, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_NEW}}
assert status == expected
status_dir = dvc.cloud.status(dir_objs, show_checksums=True)
expected = _make_dir_status(STATUS_NEW)
assert status_dir == expected
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.odb.local.cache_dir + ".backup"
move(dvc.odb.local.cache_dir, backup_dir)
status = dvc.cloud.status(foo_objs, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_MISSING}}
assert status == expected
status_dir = dvc.cloud.status(dir_objs, show_checksums=True)
expected = _make_dir_status(STATUS_MISSING)
assert status_dir == expected
# Restore original cache:
remove(dvc.odb.local.cache_dir)
move(backup_dir, dvc.odb.local.cache_dir)
# Push and check status
dvc.cloud.push(foo_objs)
assert os.path.exists(cache)
assert os.path.isfile(cache)
dvc.cloud.push(dir_objs)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_objs, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_OK}}
assert status == expected
status_dir = dvc.cloud.status(dir_objs, show_checksums=True)
expected = _make_dir_status(STATUS_OK)
assert status_dir == expected
# Remove and check status
remove(dvc.odb.local.cache_dir)
status = dvc.cloud.status(foo_objs, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
assert status == expected
status_dir = dvc.cloud.status(dir_objs, show_checksums=True)
expected = _make_dir_status(STATUS_DELETED)
assert status_dir == expected
# Pull and check status
dvc.cloud.pull(foo_objs)
assert os.path.exists(cache)
assert os.path.isfile(cache)
with open(cache) as fd:
assert fd.read() == "foo"
dvc.cloud.pull(dir_objs)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_objs, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_OK}}
assert status == expected
status_dir = dvc.cloud.status(dir_objs, show_checksums=True)
expected = _make_dir_status(STATUS_OK)
assert status_dir == expected
def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
(stage,) = tmp_dir.dvc_gen("foo", "foo")
out = stage.outs[0]
cache = out.cache_path
foo_hash = out.hash_info
foo_hashes = out.get_used_objs().get(None, set())
(stage_dir,) = tmp_dir.dvc_gen(
{
"data_dir": {
"data_sub_dir": {"data_sub": "data_sub"},
"data": "data",
"empty": "",
}
}
)
out_dir = stage_dir.outs[0]
cache_dir = out_dir.cache_path
dir_hash = out_dir.hash_info
dir_hashes = {dir_hash} | {oid for _, _, oid in out_dir.obj}
def _check_status(status, **kwargs):
for key in ("ok", "missing", "new", "deleted"):
expected = kwargs.get(key, set())
assert expected == set(getattr(status, key))
# Check status
status = dvc.cloud.status(foo_hashes)
_check_status(status, new={foo_hash})
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, new=dir_hashes)
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.odb.local.cache_dir + ".backup"
move(dvc.odb.local.cache_dir, backup_dir)
status = dvc.cloud.status(foo_hashes)
_check_status(status, missing={foo_hash})
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, missing=dir_hashes)
# Restore original cache:
remove(dvc.odb.local.cache_dir)
move(backup_dir, dvc.odb.local.cache_dir)
# Push and check status
dvc.cloud.push(foo_hashes)
assert os.path.exists(cache)
assert os.path.isfile(cache)
dvc.cloud.push(dir_hashes)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_hashes)
_check_status(status, ok={foo_hash})
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, ok=dir_hashes)
# Remove and check status
remove(dvc.odb.local.cache_dir)
status = dvc.cloud.status(foo_hashes)
_check_status(status, deleted={foo_hash})
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, deleted=dir_hashes)
# Pull and check status
dvc.cloud.pull(foo_hashes)
assert os.path.exists(cache)
assert os.path.isfile(cache)
with open(cache, encoding="utf-8") as fd:
assert fd.read() == "foo"
dvc.cloud.pull(dir_hashes)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_hashes)
_check_status(status, ok={foo_hash})
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, ok=dir_hashes)
def move(self, from_info, to_info):
self.makedirs(self.path.parent(to_info))
move(from_info, to_info)
def mv(self, path1, path2, **kwargs):
self.makedirs(self._parent(path2), exist_ok=True)
move(path1, path2)
def move(self, from_info, to_info):
if from_info.scheme != "local" or to_info.scheme != "local":
raise NotImplementedError
self.makedirs(to_info.parent)
move(from_info, to_info)
def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
(stage,) = tmp_dir.dvc_gen("foo", "foo")
out = stage.outs[0]
cache = out.cache_path
md5 = out.checksum
info = out.get_used_cache()
(stage_dir,) = tmp_dir.dvc_gen(
{
"data_dir": {
"data_sub_dir": {"data_sub": "data_sub"},
"data": "data",
}
}
)
out_dir = stage_dir.outs[0]
cache_dir = out_dir.cache_path
name_dir = str(out_dir)
md5_dir = out_dir.checksum
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
with dvc.state:
# Check status
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_NEW}}
assert status == expected
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
assert status_dir == expected
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.cache.local.cache_dir + ".backup"
move(dvc.cache.local.cache_dir, backup_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_MISSING}}
assert status == expected
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
expected = {md5_dir: {"name": md5_dir, "status": STATUS_MISSING}}
assert status_dir == expected
# Restore original cache:
remove(dvc.cache.local.cache_dir)
move(backup_dir, dvc.cache.local.cache_dir)
# Push and check status
dvc.cloud.push(info)
assert os.path.exists(cache)
assert os.path.isfile(cache)
dvc.cloud.push(info_dir)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_OK}}
assert status == expected
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
assert status_dir == expected
# Remove and check status
remove(dvc.cache.local.cache_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_DELETED}}
assert status == expected
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
assert status_dir == expected
# Pull and check status
dvc.cloud.pull(info)
assert os.path.exists(cache)
assert os.path.isfile(cache)
with open(cache) as fd:
assert fd.read() == "foo"
dvc.cloud.pull(info_dir)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(info, show_checksums=True)
expected = {md5: {"name": md5, "status": STATUS_OK}}
assert status == expected
status_dir = dvc.cloud.status(info_dir, show_checksums=True)
expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
assert status_dir == expected
