def collect(self, arg): path, local = arg ret = [path] if not RemoteLOCAL.is_dir_cache(path): return ret if local: if not os.path.isfile(path): return ret dir_path = path else: key = self._get_key(path) if not key: Logger.debug( "File '{}' does not exist in the cloud".format(path)) return ret tmp = os.path.join(tempfile.mkdtemp(), os.path.basename(path)) self._pull_key(key, tmp, no_progress_bar=True) dir_path = tmp for relpath, md5 in RemoteLOCAL.get_dir_cache(dir_path).items(): cache = self._cloud_settings.cache.local.get(md5) ret.append(cache) return ret
def test_status_download_optimization(mocker): """When comparing the status to pull a remote cache, And the desired files to fetch are already on the local cache, Don't check the existance of the desired files on the remote cache """ remote = RemoteLOCAL(None, {}) infos = [ { "name": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8" }, { "name": "bar", "md5": "37b51d194a7513e45b56f6524f2d51f2" }, ] local_exists = [ "acbd18db4cc2f85cedef654fccc4a4d8", "37b51d194a7513e45b56f6524f2d51f2", ] mocker.patch.object(remote, "cache_exists", return_value=local_exists) other_remote = mocker.Mock() other_remote.url = "other_remote" other_remote.cache_exists.return_value = [] remote.status(infos, other_remote, download=True) assert other_remote.cache_exists.call_count == 0
class DependencyLOCAL(DependencyBase): REGEX = r'^(?P<path>(/+|.:\\+)?.*)$' DoesNotExistError = DependencyDoesNotExistError IsNotFileOrDirError = DependencyIsNotFileOrDirError def __init__(self, stage, path, info=None): self.stage = stage self.project = stage.project if not os.path.isabs(path): path = self.unixpath(path) path = os.path.join(stage.cwd, path) self.path = os.path.normpath(path) self.info = info self.remote = RemoteLOCAL(stage.project, {Config.SECTION_REMOTE_URL: self.project.dvc_dir}) self.path_info = {'scheme': 'local', 'path': self.path} @property def rel_path(self): return os.path.relpath(self.path) def changed(self): if not os.path.exists(self.path): return True info = self.remote.save_info(self.path_info) return self.info != info def save(self): if not os.path.exists(self.path): raise self.DoesNotExistError(self.rel_path) if not os.path.isfile(self.path) and not os.path.isdir(self.path): raise self.NotFileOrDirError(self.rel_path) if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): self.project.logger.warn("File/directory '{}' is empty.".format(self.rel_path)) self.info = self.remote.save_info(self.path_info) @staticmethod def unixpath(path): assert not ntpath.isabs(path) assert not posixpath.isabs(path) return path.replace('\\', '/') def dumpd(self): if self.path.startswith(self.stage.project.root_dir): path = self.unixpath(os.path.relpath(self.path, self.stage.cwd)) else: path = self.path info = self.info info[self.PARAM_PATH] = path return info
def test(self): fname = 'not-json' self.create(fname, '<clearly>not,json') self._do_test(RemoteLOCAL.load_dir_cache(fname)) fname = 'not-list' self.create(fname, '{"a": "b"}') self._do_test(RemoteLOCAL.load_dir_cache(fname))
def test_protect_ignore_erofs(tmp_dir, mocker): tmp_dir.gen("foo", "foo") foo = PathInfo("foo") remote = RemoteLOCAL(None, {}) mock_chmod = mocker.patch("os.chmod", side_effect=OSError(errno.EROFS, "read-only fs")) remote.protect(foo) assert mock_chmod.called
def __init__(self, stage, path, info=None): self.stage = stage self.project = stage.project if not os.path.isabs(path): path = self.unixpath(path) path = os.path.join(stage.cwd, path) self.path = os.path.normpath(path) self.info = info self.remote = RemoteLOCAL(stage.project, {Config.SECTION_REMOTE_URL: self.project.dvc_dir}) self.path_info = {'scheme': 'local', 'path': self.path}
def __init__(self, stage, path, info=None, remote=None): self.stage = stage self.project = stage.project self.info = info if remote is not None: self.remote = remote else: self.remote = RemoteLOCAL(stage.project, {}) if remote: path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/')) if not os.path.isabs(path): path = self.remote.ospath(path) path = os.path.join(stage.cwd, path) self.path = os.path.abspath(os.path.normpath(path)) self.path_info = {'scheme': 'local', 'path': self.path}
def __init__(self, stage, path, info=None, remote=None): self.stage = stage self.project = stage.project if not os.path.isabs(path): path = self.unixpath(path) path = os.path.join(stage.cwd, path) self.path = os.path.normpath(path) self.info = info self.remote = remote if remote != None else RemoteLOCAL( stage.project, {}) self.path_info = {'scheme': 'local', 'path': self.path}
def __init__(self, stage, path, info=None, remote=None): super(DependencyLOCAL, self).__init__(stage, path, info) if remote is not None: self.remote = remote else: self.remote = RemoteLOCAL(stage.project, {}) if remote: p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip('/')) else: p = path if not os.path.isabs(p): p = self.remote.to_ospath(p) p = os.path.join(stage.cwd, p) p = os.path.abspath(os.path.normpath(p)) self.path_info = {'scheme': 'local', 'path': p}
def test_status_download_optimization(mocker, dvc): """When comparing the status to pull a remote cache, And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ remote = RemoteLOCAL(dvc, {}) infos = NamedCache() infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo") infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar") local_exists = list(infos["local"]) mocker.patch.object(remote, "cache_exists", return_value=local_exists) other_remote = mocker.Mock() other_remote.url = "other_remote" other_remote.cache_exists.return_value = [] remote.status(infos, other_remote, download=True) assert other_remote.cache_exists.call_count == 0
def test_protect_ignore_errors(tmp_dir, mocker, err): tmp_dir.gen("foo", "foo") foo = PathInfo("foo") remote = RemoteLOCAL(None, {}) remote.protect(foo) mock_chmod = mocker.patch("os.chmod", side_effect=OSError(err, "something")) remote.protect(foo) assert mock_chmod.called
def dumpd(self): from dvc.remote.local import RemoteLOCAL return { key: value for key, value in { Stage.PARAM_MD5: self.md5, Stage.PARAM_CMD: self.cmd, Stage.PARAM_WDIR: RemoteLOCAL.unixpath( os.path.relpath(self.wdir, os.path.dirname(self.path)) ), Stage.PARAM_LOCKED: self.locked, Stage.PARAM_DEPS: [d.dumpd() for d in self.deps], Stage.PARAM_OUTS: [o.dumpd() for o in self.outs], }.items() if value }
class DependencyLOCAL(DependencyBase): REGEX = r'^(?P<path>(/+|.:\\+)?[^:]*)$' DoesNotExistError = DependencyDoesNotExistError IsNotFileOrDirError = DependencyIsNotFileOrDirError def __init__(self, stage, path, info=None, remote=None): self.stage = stage self.project = stage.project self.info = info if remote is not None: self.remote = remote else: self.remote = RemoteLOCAL(stage.project, {}) if remote: path = os.path.join(remote.prefix, urlparse(path).path.lstrip('/')) if not os.path.isabs(path): path = self.remote.ospath(path) path = os.path.join(stage.cwd, path) self.path = os.path.abspath(os.path.normpath(path)) self.path_info = {'scheme': 'local', 'path': self.path} @property def sep(self): return os.sep @property def rel_path(self): return os.path.relpath(self.path) def changed(self): if not self.exists: return True info = self.remote.save_info(self.path_info) return self.info != info def save(self): if not self.exists: raise self.DoesNotExistError(self.rel_path) if not os.path.isfile(self.path) and not os.path.isdir(self.path): raise self.IsNotFileOrDirError(self.rel_path) if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): Logger.warn("File/directory '{}' is empty.".format(self.rel_path)) self.info = self.remote.save_info(self.path_info) def dumpd(self): if self.path.startswith(self.stage.project.root_dir): path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd)) else: path = self.path info = self.info.copy() info[self.PARAM_PATH] = path return info
def test_is_protected(tmp_dir, dvc, link_name): remote = RemoteLOCAL(dvc, {}) link_method = getattr(remote, link_name) (tmp_dir / "foo").write_text("foo") foo = PathInfo(tmp_dir / "foo") link = PathInfo(tmp_dir / "link") link_method(foo, link) assert not remote.is_protected(foo) assert not remote.is_protected(link) remote.protect(foo) assert remote.is_protected(foo) assert remote.is_protected(link) remote.unprotect(link) assert not remote.is_protected(link) if os.name == "nt" and link_name == "hardlink": # NOTE: NTFS doesn't allow deleting read-only files, which forces us to # set write perms on the link, which propagates to the source. assert not remote.is_protected(foo) else: assert remote.is_protected(foo)
def test_is_protected(tmp_dir, link_name): remote = RemoteLOCAL(None, {}) link_method = getattr(remote, link_name) (tmp_dir / "foo").write_text("foo") foo = PathInfo(tmp_dir / "foo") link = PathInfo(tmp_dir / "link") link_method(foo, link) assert not remote.is_protected(foo) assert not remote.is_protected(link) remote.protect(foo) assert remote.is_protected(foo) assert remote.is_protected(link) remote.unprotect(link) assert not remote.is_protected(link) if link_name == "symlink" and os.name == "nt": # NOTE: Windows symlink perms don't propagate to the target assert remote.is_protected(foo) else: assert not remote.is_protected(foo)
class DependencyLOCAL(DependencyBase): REGEX = r'^(?P<path>.*)$' DoesNotExistError = DependencyDoesNotExistError IsNotFileOrDirError = DependencyIsNotFileOrDirError def __init__(self, stage, path, info=None, remote=None): super(DependencyLOCAL, self).__init__(stage, path, info) if remote is not None: self.remote = remote else: self.remote = RemoteLOCAL(stage.project, {}) if remote: p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip('/')) else: p = path if not os.path.isabs(p): p = self.remote.to_ospath(p) p = os.path.join(stage.cwd, p) p = os.path.abspath(os.path.normpath(p)) self.path_info = {'scheme': 'local', 'path': p} def __str__(self): return self.rel_path @property def is_local(self): return (urlparse(self.url).scheme != 'remote' and not os.path.isabs(self.url)) @property def sep(self): return os.sep @property def rel_path(self): return os.path.relpath(self.path) def changed(self): if not self.exists: return True info = self.remote.save_info(self.path_info) return self.info != info def save(self): if not self.exists: raise self.DoesNotExistError(self.rel_path) if not os.path.isfile(self.path) \ and not os.path.isdir(self.path): # pragma: no cover raise self.IsNotFileOrDirError(self.rel_path) if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): msg = "file/directory '{}' is empty.".format(self.rel_path) logger.warning(msg) self.info = self.remote.save_info(self.path_info) def dumpd(self): if self.is_local: path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd)) else: path = self.url info = self.info.copy() info[self.PARAM_PATH] = path return info