def collect(self, arg):
path, local = arg
ret = [path]
if not RemoteLOCAL.is_dir_cache(path):
return ret
if local:
if not os.path.isfile(path):
return ret
dir_path = path
else:
key = self._get_key(path)
if not key:
Logger.debug(
"File '{}' does not exist in the cloud".format(path))
return ret
tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path))
self._pull_key(key, tmp, no_progress_bar=True)
dir_path = tmp
for relpath, md5 in RemoteLOCAL.get_dir_cache(dir_path).items():
cache = self._cloud_settings.cache.local.get(md5)
ret.append(cache)
return ret
def test_status_download_optimization(mocker):
"""When comparing the status to pull a remote cache,
And the desired files to fetch are already on the local cache,
Don't check the existance of the desired files on the remote cache
"""
remote = RemoteLOCAL(None, {})
infos = [
{
"name": "foo",
"md5": "acbd18db4cc2f85cedef654fccc4a4d8"
},
{
"name": "bar",
"md5": "37b51d194a7513e45b56f6524f2d51f2"
},
]
local_exists = [
"acbd18db4cc2f85cedef654fccc4a4d8",
"37b51d194a7513e45b56f6524f2d51f2",
]
mocker.patch.object(remote, "cache_exists", return_value=local_exists)
other_remote = mocker.Mock()
other_remote.url = "other_remote"
other_remote.cache_exists.return_value = []
remote.status(infos, other_remote, download=True)
assert other_remote.cache_exists.call_count == 0
class DependencyLOCAL(DependencyBase):
REGEX = r'^(?P<path>(/+|.:\\+)?.*)$'
DoesNotExistError = DependencyDoesNotExistError
IsNotFileOrDirError = DependencyIsNotFileOrDirError
def __init__(self, stage, path, info=None):
self.stage = stage
self.project = stage.project
if not os.path.isabs(path):
path = self.unixpath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.normpath(path)
self.info = info
self.remote = RemoteLOCAL(stage.project,
{Config.SECTION_REMOTE_URL: self.project.dvc_dir})
self.path_info = {'scheme': 'local',
'path': self.path}
@property
def rel_path(self):
return os.path.relpath(self.path)
def changed(self):
if not os.path.exists(self.path):
return True
info = self.remote.save_info(self.path_info)
return self.info != info
def save(self):
if not os.path.exists(self.path):
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
raise self.NotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path))
self.info = self.remote.save_info(self.path_info)
@staticmethod
def unixpath(path):
assert not ntpath.isabs(path)
assert not posixpath.isabs(path)
return path.replace('\\', '/')
def dumpd(self):
if self.path.startswith(self.stage.project.root_dir):
path = self.unixpath(os.path.relpath(self.path, self.stage.cwd))
else:
path = self.path
info = self.info
info[self.PARAM_PATH] = path
return info
def test(self):
fname = 'not-json'
self.create(fname, '<clearly>not,json')
self._do_test(RemoteLOCAL.load_dir_cache(fname))
fname = 'not-list'
self.create(fname, '{"a": "b"}')
self._do_test(RemoteLOCAL.load_dir_cache(fname))
def test_protect_ignore_erofs(tmp_dir, mocker):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
remote = RemoteLOCAL(None, {})
mock_chmod = mocker.patch("os.chmod",
side_effect=OSError(errno.EROFS, "read-only fs"))
remote.protect(foo)
assert mock_chmod.called
def __init__(self, stage, path, info=None):
self.stage = stage
self.project = stage.project
if not os.path.isabs(path):
path = self.unixpath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.normpath(path)
self.info = info
self.remote = RemoteLOCAL(stage.project,
{Config.SECTION_REMOTE_URL: self.project.dvc_dir})
self.path_info = {'scheme': 'local',
'path': self.path}
def __init__(self, stage, path, info=None, remote=None):
self.stage = stage
self.project = stage.project
self.info = info
if remote is not None:
self.remote = remote
else:
self.remote = RemoteLOCAL(stage.project, {})
if remote:
path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
if not os.path.isabs(path):
path = self.remote.ospath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.abspath(os.path.normpath(path))
self.path_info = {'scheme': 'local', 'path': self.path}
def __init__(self, stage, path, info=None, remote=None):
self.stage = stage
self.project = stage.project
if not os.path.isabs(path):
path = self.unixpath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.normpath(path)
self.info = info
self.remote = remote if remote != None else RemoteLOCAL(
stage.project, {})
self.path_info = {'scheme': 'local', 'path': self.path}
def __init__(self, stage, path, info=None, remote=None):
super(DependencyLOCAL, self).__init__(stage, path, info)
if remote is not None:
self.remote = remote
else:
self.remote = RemoteLOCAL(stage.project, {})
if remote:
p = os.path.join(remote.prefix,
urlparse(self.url).path.lstrip('/'))
else:
p = path
if not os.path.isabs(p):
p = self.remote.to_ospath(p)
p = os.path.join(stage.cwd, p)
p = os.path.abspath(os.path.normpath(p))
self.path_info = {'scheme': 'local',
'path': p}
def test_status_download_optimization(mocker, dvc):
"""When comparing the status to pull a remote cache,
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
remote = RemoteLOCAL(dvc, {})
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
local_exists = list(infos["local"])
mocker.patch.object(remote, "cache_exists", return_value=local_exists)
other_remote = mocker.Mock()
other_remote.url = "other_remote"
other_remote.cache_exists.return_value = []
remote.status(infos, other_remote, download=True)
assert other_remote.cache_exists.call_count == 0
def test_protect_ignore_errors(tmp_dir, mocker, err):
tmp_dir.gen("foo", "foo")
foo = PathInfo("foo")
remote = RemoteLOCAL(None, {})
remote.protect(foo)
mock_chmod = mocker.patch("os.chmod",
side_effect=OSError(err, "something"))
remote.protect(foo)
assert mock_chmod.called
def dumpd(self):
from dvc.remote.local import RemoteLOCAL
return {
key: value
for key, value in {
Stage.PARAM_MD5: self.md5,
Stage.PARAM_CMD: self.cmd,
Stage.PARAM_WDIR: RemoteLOCAL.unixpath(
os.path.relpath(self.wdir, os.path.dirname(self.path))
),
Stage.PARAM_LOCKED: self.locked,
Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
}.items()
if value
}
class DependencyLOCAL(DependencyBase):
REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$'
DoesNotExistError = DependencyDoesNotExistError
IsNotFileOrDirError = DependencyIsNotFileOrDirError
def __init__(self, stage, path, info=None, remote=None):
self.stage = stage
self.project = stage.project
self.info = info
if remote is not None:
self.remote = remote
else:
self.remote = RemoteLOCAL(stage.project, {})
if remote:
path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/'))
if not os.path.isabs(path):
path = self.remote.ospath(path)
path = os.path.join(stage.cwd, path)
self.path = os.path.abspath(os.path.normpath(path))
self.path_info = {'scheme': 'local',
'path': self.path}
@property
def sep(self):
return os.sep
@property
def rel_path(self):
return os.path.relpath(self.path)
def changed(self):
if not self.exists:
return True
info = self.remote.save_info(self.path_info)
return self.info != info
def save(self):
if not self.exists:
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) and not os.path.isdir(self.path):
raise self.IsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
Logger.warn("File/directory '{}' is empty.".format(self.rel_path))
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
if self.path.startswith(self.stage.project.root_dir):
path = self.remote.unixpath(os.path.relpath(self.path,
self.stage.cwd))
else:
path = self.path
info = self.info.copy()
info[self.PARAM_PATH] = path
return info
def test_is_protected(tmp_dir, dvc, link_name):
remote = RemoteLOCAL(dvc, {})
link_method = getattr(remote, link_name)
(tmp_dir / "foo").write_text("foo")
foo = PathInfo(tmp_dir / "foo")
link = PathInfo(tmp_dir / "link")
link_method(foo, link)
assert not remote.is_protected(foo)
assert not remote.is_protected(link)
remote.protect(foo)
assert remote.is_protected(foo)
assert remote.is_protected(link)
remote.unprotect(link)
assert not remote.is_protected(link)
if os.name == "nt" and link_name == "hardlink":
# NOTE: NTFS doesn't allow deleting read-only files, which forces us to
# set write perms on the link, which propagates to the source.
assert not remote.is_protected(foo)
else:
assert remote.is_protected(foo)
def test_is_protected(tmp_dir, link_name):
remote = RemoteLOCAL(None, {})
link_method = getattr(remote, link_name)
(tmp_dir / "foo").write_text("foo")
foo = PathInfo(tmp_dir / "foo")
link = PathInfo(tmp_dir / "link")
link_method(foo, link)
assert not remote.is_protected(foo)
assert not remote.is_protected(link)
remote.protect(foo)
assert remote.is_protected(foo)
assert remote.is_protected(link)
remote.unprotect(link)
assert not remote.is_protected(link)
if link_name == "symlink" and os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
assert remote.is_protected(foo)
else:
assert not remote.is_protected(foo)
class DependencyLOCAL(DependencyBase):
REGEX = r'^(?P<path>.*)$'
DoesNotExistError = DependencyDoesNotExistError
IsNotFileOrDirError = DependencyIsNotFileOrDirError
def __init__(self, stage, path, info=None, remote=None):
super(DependencyLOCAL, self).__init__(stage, path, info)
if remote is not None:
self.remote = remote
else:
self.remote = RemoteLOCAL(stage.project, {})
if remote:
p = os.path.join(remote.prefix,
urlparse(self.url).path.lstrip('/'))
else:
p = path
if not os.path.isabs(p):
p = self.remote.to_ospath(p)
p = os.path.join(stage.cwd, p)
p = os.path.abspath(os.path.normpath(p))
self.path_info = {'scheme': 'local',
'path': p}
def __str__(self):
return self.rel_path
@property
def is_local(self):
return (urlparse(self.url).scheme != 'remote'
and not os.path.isabs(self.url))
@property
def sep(self):
return os.sep
@property
def rel_path(self):
return os.path.relpath(self.path)
def changed(self):
if not self.exists:
return True
info = self.remote.save_info(self.path_info)
return self.info != info
def save(self):
if not self.exists:
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) \
and not os.path.isdir(self.path): # pragma: no cover
raise self.IsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
msg = "file/directory '{}' is empty.".format(self.rel_path)
logger.warning(msg)
self.info = self.remote.save_info(self.path_info)
def dumpd(self):
if self.is_local:
path = self.remote.unixpath(os.path.relpath(self.path,
self.stage.cwd))
else:
path = self.url
info = self.info.copy()
info[self.PARAM_PATH] = path
return info
