def __init__(self, repo): from dvc.remote import Remote self.repo = repo config = repo.config.config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL) if local: name = Config.SECTION_REMOTE_FMT.format(local) settings = repo.config.config[name] else: default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR) cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) cache_type = config.get(Config.SECTION_CACHE_TYPE) protected = config.get(Config.SECTION_CACHE_PROTECTED) settings = { Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, repo.dvc_dir), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, Config.SECTION_CACHE_PROTECTED: protected, } self.local = Remote(repo, **settings) self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def _get_remote(self, project, config, name): remote = config.get(name, None) if not remote: return None sect = project.config._config[Config.SECTION_REMOTE_FMT.format(remote)] return Remote(project, sect)
def __init__(self, project): from dvc.remote import Remote config = project.config._config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL, None) if local: sect = project.config._config[Config.SECTION_REMOTE_FMT.format( local)] else: sect = {} cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR) if not os.path.isabs(cache_dir): cache_dir = os.path.abspath( os.path.realpath(os.path.join(project.dvc_dir, cache_dir))) sect[Config.SECTION_REMOTE_URL] = cache_dir t = config.get(Config.SECTION_CACHE_TYPE, None) if t: sect[Config.SECTION_CACHE_TYPE] = t self.local = Remote(project, sect) self.s3 = self._get_remote(project, config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(project, config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(project, config, Config.SECTION_CACHE_SSH) self.hdfs = self._get_remote(project, config, Config.SECTION_CACHE_HDFS)
def _get(stage, p, info, cache, metric, persist): parsed = urlparse(p) if parsed.scheme == "remote": name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) sect = stage.repo.config.config[name] remote = Remote(stage.repo, sect) return OUTS_MAP[remote.scheme]( stage, p, info, cache=cache, remote=remote, metric=metric, persist=persist, ) for o in OUTS: if o.supported(p): return o(stage, p, info, cache=cache, remote=None, metric=metric) return OutputLOCAL( stage, p, info, cache=cache, remote=None, metric=metric, persist=persist, )
def test_remote_without_checksum_jobs_default(dvc): set_config_opts( dvc, [('remote "without_checksum_jobs"', "url", "s3://bucket/name")] ) remote = Remote(dvc, name="without_checksum_jobs") assert remote.checksum_jobs == remote.CHECKSUM_JOBS
def __init__(self, project): from dvc.remote import Remote self.project = project config = project.config._config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL, None) if local: name = Config.SECTION_REMOTE_FMT.format(local) sect = project.config._config[name] else: sect = {} cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR) if not os.path.isabs(cache_dir): cache_dir = os.path.join(project.dvc_dir, cache_dir) cache_dir = os.path.abspath(os.path.realpath(cache_dir)) sect[Config.SECTION_REMOTE_URL] = cache_dir t = config.get(Config.SECTION_CACHE_TYPE, None) if t: sect[Config.SECTION_CACHE_TYPE] = t protected = config.get(Config.SECTION_CACHE_PROTECTED, None) if protected is not None: sect[Config.SECTION_CACHE_PROTECTED] = protected self.local = Remote(project, sect) self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def __init__(self, project): from dvc.remote import Remote self.project = project config = project.config.config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL) if local: name = Config.SECTION_REMOTE_FMT.format(local) sect = project.config.config[name] else: default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR) cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) cache_type = config.get(Config.SECTION_CACHE_TYPE) protected = config.get(Config.SECTION_CACHE_PROTECTED) sect = { Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, Config.SECTION_CACHE_PROTECTED: protected, } self._local = Remote(project, sect) self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self._gs = self._get_remote(config, Config.SECTION_CACHE_GS) self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS) self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def _get(stage, p, info, cache, metric, persist=False): parsed = urlparse(p) if parsed.scheme == "remote": remote = Remote(stage.repo, name=parsed.netloc) return OUTS_MAP[remote.scheme]( stage, p, info, cache=cache, remote=remote, metric=metric, persist=persist, ) for o in OUTS: if o.supported(p): return o( stage, p, info, cache=cache, remote=None, metric=metric, persist=persist, ) return LocalOutput( stage, p, info, cache=cache, remote=None, metric=metric, persist=persist, )
def __init__(self, repo): from dvc.remote import Remote self.repo = repo self.config = config = repo.config.config[Config.SECTION_CACHE] local = config.get(Config.SECTION_CACHE_LOCAL) if local: name = Config.SECTION_REMOTE_FMT.format(local) settings = repo.config.config[name] else: default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR) cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) cache_type = config.get(Config.SECTION_CACHE_TYPE) protected = config.get(Config.SECTION_CACHE_PROTECTED) shared = config.get(Config.SECTION_CACHE_SHARED) settings = { Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, repo.dvc_dir), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, Config.SECTION_CACHE_PROTECTED: protected, Config.SECTION_CACHE_SHARED: shared, } self.local = Remote(repo, **settings)
def _init_remote(self, remote): section = Config.SECTION_REMOTE_FMT.format(remote).lower() cloud_config = self._config.get(section, None) if not cloud_config: msg = "can't find remote section '{}' in config" raise ConfigError(msg.format(section)) return Remote(self.repo, cloud_config)
def getter(self): from dvc.remote import Remote remote = self.config.get(name) if not remote: return None return Remote(self.repo, name=remote)
def test_remote_with_checksum_jobs(dvc): dvc.config["remote"]["with_checksum_jobs"] = { "url": "s3://bucket/name", "checksum_jobs": 100, } dvc.config["core"]["checksum_jobs"] = 200 remote = Remote(dvc, name="with_checksum_jobs") assert remote.checksum_jobs == 100
def _get_remote(self, config, name): from dvc.remote import Remote remote = config.get(name, None) if not remote: return None name = Config.SECTION_REMOTE_FMT.format(remote) sect = self.project.config._config[name] return Remote(self.project, sect)
def test_remote_without_checksum_jobs(dvc): set_config_opts( dvc, [ ('remote "without_checksum_jobs"', "url", "s3://bucket/name"), ("core", "checksum_jobs", "200"), ], ) remote = Remote(dvc, name="without_checksum_jobs") assert remote.checksum_jobs == 200
def _get(stage, p, info, cache, metric): parsed = urlparse(p) if parsed.scheme == 'remote': sect = stage.project.config._config[Config.SECTION_REMOTE_FMT.format(parsed.netloc)] remote = Remote(stage.project, sect) return OUTS_MAP[remote.scheme](stage, p, info, cache=cache, remote=remote, metric=metric) for o in OUTS: if o.supported(p): return o(stage, p, info, cache=cache, remote=None, metric=metric) raise DvcException('Output \'{}\' is not supported'.format(p))
def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == 'remote': sect = stage.project.config._config[Config.SECTION_REMOTE_FMT.format(parsed.netloc)] remote = Remote(stage.project, sect) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: if d.supported(p): return d(stage, p, info) raise DvcException('Dependency \'{}\' is not supported'.format(p))
def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == 'remote': name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) sect = stage.project.config.config[name] remote = Remote(stage.project, sect) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: if d.supported(p): return d(stage, p, info) return DependencyLOCAL(stage, p, info)
def _get(stage, p, info): from dvc.utils.compat import urlparse parsed = urlparse(p) if parsed.scheme == "remote": remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: if d.supported(p): return d(stage, p, info) return DependencyLOCAL(stage, p, info)
def __init__(self, repo): from dvc.remote import Remote self.repo = repo self.config = config = repo.config["cache"] local = config.get("local") if local: settings = {"name": local} else: settings = {**config, "url": config["dir"]} self.local = Remote(repo, **settings)
def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == "remote": remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) if info and info.get(DependencyREPO.PARAM_REPO): repo = info.pop(DependencyREPO.PARAM_REPO) return DependencyREPO(repo, stage, p, info) for d in DEPS: if d.supported(p): return d(stage, p, info) return DependencyLOCAL(stage, p, info)
def _get(stage, p, info): from dvc.utils.compat import urlparse parsed = urlparse(p) if parsed.scheme == "remote": remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) if info and info.get(DependencyPKG.PARAM_PKG): pkg = info.pop(DependencyPKG.PARAM_PKG) return DependencyPKG(pkg, stage, p, info) for d in DEPS: if d.supported(p): return d(stage, p, info) return DependencyLOCAL(stage, p, info)
def _get(stage, p, info): parsed = urlparse(p) if p else None if parsed and parsed.scheme == "remote": remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) if info and info.get(RepoDependency.PARAM_REPO): repo = info.pop(RepoDependency.PARAM_REPO) return RepoDependency(repo, stage, p, info) if info and info.get(ParamsDependency.PARAM_PARAMS): params = info.pop(ParamsDependency.PARAM_PARAMS) return ParamsDependency(stage, p, params) for d in DEPS: if d.supported(p): return d(stage, p, info) return LocalDependency(stage, p, info)
def _get_remote(self, config, name): """ The config file is stored in a way that allows you to have a cache for each remote. This is needed when specifying external outputs (as they require you to have an external cache location). Imagine a config file like the following: ['remote "dvc-storage"'] url = ssh://localhost/tmp ask_password = true [cache] ssh = dvc-storage This method resolves the name under the cache section into the correct Remote instance. Args: config (dict): The cache section on the config file name (str): Name of the section we are interested in to retrieve Returns: remote (dvc.Remote): Remote instance that the section is referring. None when there's no remote with that name. Example: >>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh') """ from dvc.remote import Remote remote = config.get(name) if not remote: return None settings = self.repo.config.get_remote_settings(remote) return Remote(self.repo, settings)
def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) if parsed.scheme == "remote": settings = stage.repo.config.get_remote_settings(parsed.netloc) remote = Remote(stage.repo, settings) return OUTS_MAP[remote.scheme]( stage, p, info, cache=cache, remote=remote, metric=metric, persist=persist, tags=tags, ) for o in OUTS: if o.supported(p): return o( stage, p, info, cache=cache, remote=None, metric=metric, persist=persist, tags=tags, ) return OutputLOCAL( stage, p, info, cache=cache, remote=None, metric=metric, persist=persist, tags=tags, )
def _init_remote(self, name): return Remote(self.repo, name=name)
def _init_remote(self, remote): config = self.repo.config.get_remote_settings(remote) return Remote(self.repo, config)
def test_unsupported(self): with self.assertRaises(UnsupportedRemoteError): Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})
def test_remote_without_checksum_jobs_default(dvc): dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"} remote = Remote(dvc, name="without_checksum_jobs") assert remote.checksum_jobs == remote.CHECKSUM_JOBS