def __init__(self, repo):
from dvc.remote import Remote
self.repo = repo
config = repo.config.config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL)
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
settings = repo.config.config[name]
else:
default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
cache_type = config.get(Config.SECTION_CACHE_TYPE)
protected = config.get(Config.SECTION_CACHE_PROTECTED)
settings = {
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
repo.dvc_dir),
Config.SECTION_REMOTE_URL: cache_dir,
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_PROTECTED: protected,
}
self.local = Remote(repo, **settings)
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def _get_remote(self, project, config, name):
remote = config.get(name, None)
if not remote:
return None
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(remote)]
return Remote(project, sect)
def __init__(self, project):
from dvc.remote import Remote
config = project.config._config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL, None)
if local:
sect = project.config._config[Config.SECTION_REMOTE_FMT.format(
local)]
else:
sect = {}
cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
if not os.path.isabs(cache_dir):
cache_dir = os.path.abspath(
os.path.realpath(os.path.join(project.dvc_dir, cache_dir)))
sect[Config.SECTION_REMOTE_URL] = cache_dir
t = config.get(Config.SECTION_CACHE_TYPE, None)
if t:
sect[Config.SECTION_CACHE_TYPE] = t
self.local = Remote(project, sect)
self.s3 = self._get_remote(project, config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(project, config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(project, config, Config.SECTION_CACHE_SSH)
self.hdfs = self._get_remote(project, config,
Config.SECTION_CACHE_HDFS)
def _get(stage, p, info, cache, metric, persist):
parsed = urlparse(p)
if parsed.scheme == "remote":
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
sect = stage.repo.config.config[name]
remote = Remote(stage.repo, sect)
return OUTS_MAP[remote.scheme](
stage,
p,
info,
cache=cache,
remote=remote,
metric=metric,
persist=persist,
)
for o in OUTS:
if o.supported(p):
return o(stage, p, info, cache=cache, remote=None, metric=metric)
return OutputLOCAL(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
persist=persist,
)
def test_remote_without_checksum_jobs_default(dvc):
set_config_opts(
dvc, [('remote "without_checksum_jobs"', "url", "s3://bucket/name")]
)
remote = Remote(dvc, name="without_checksum_jobs")
assert remote.checksum_jobs == remote.CHECKSUM_JOBS
def __init__(self, project):
from dvc.remote import Remote
self.project = project
config = project.config._config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL, None)
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
sect = project.config._config[name]
else:
sect = {}
cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR)
if not os.path.isabs(cache_dir):
cache_dir = os.path.join(project.dvc_dir, cache_dir)
cache_dir = os.path.abspath(os.path.realpath(cache_dir))
sect[Config.SECTION_REMOTE_URL] = cache_dir
t = config.get(Config.SECTION_CACHE_TYPE, None)
if t:
sect[Config.SECTION_CACHE_TYPE] = t
protected = config.get(Config.SECTION_CACHE_PROTECTED, None)
if protected is not None:
sect[Config.SECTION_CACHE_PROTECTED] = protected
self.local = Remote(project, sect)
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
self.hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
self.azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def __init__(self, project):
from dvc.remote import Remote
self.project = project
config = project.config.config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL)
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
sect = project.config.config[name]
else:
default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR)
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
cache_type = config.get(Config.SECTION_CACHE_TYPE)
protected = config.get(Config.SECTION_CACHE_PROTECTED)
sect = {
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
project.dvc_dir),
Config.SECTION_REMOTE_URL: cache_dir,
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_PROTECTED: protected,
}
self._local = Remote(project, sect)
self._s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self._gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self._ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
self._hdfs = self._get_remote(config, Config.SECTION_CACHE_HDFS)
self._azure = self._get_remote(config, Config.SECTION_CACHE_AZURE)
def _get(stage, p, info, cache, metric, persist=False):
parsed = urlparse(p)
if parsed.scheme == "remote":
remote = Remote(stage.repo, name=parsed.netloc)
return OUTS_MAP[remote.scheme](
stage,
p,
info,
cache=cache,
remote=remote,
metric=metric,
persist=persist,
)
for o in OUTS:
if o.supported(p):
return o(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
persist=persist,
)
return LocalOutput(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
persist=persist,
)
def __init__(self, repo):
from dvc.remote import Remote
self.repo = repo
self.config = config = repo.config.config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL)
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
settings = repo.config.config[name]
else:
default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
cache_type = config.get(Config.SECTION_CACHE_TYPE)
protected = config.get(Config.SECTION_CACHE_PROTECTED)
shared = config.get(Config.SECTION_CACHE_SHARED)
settings = {
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
repo.dvc_dir),
Config.SECTION_REMOTE_URL: cache_dir,
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_PROTECTED: protected,
Config.SECTION_CACHE_SHARED: shared,
}
self.local = Remote(repo, **settings)
def _init_remote(self, remote):
section = Config.SECTION_REMOTE_FMT.format(remote).lower()
cloud_config = self._config.get(section, None)
if not cloud_config:
msg = "can't find remote section '{}' in config"
raise ConfigError(msg.format(section))
return Remote(self.repo, cloud_config)
def getter(self):
from dvc.remote import Remote
remote = self.config.get(name)
if not remote:
return None
return Remote(self.repo, name=remote)
def test_remote_with_checksum_jobs(dvc):
dvc.config["remote"]["with_checksum_jobs"] = {
"url": "s3://bucket/name",
"checksum_jobs": 100,
}
dvc.config["core"]["checksum_jobs"] = 200
remote = Remote(dvc, name="with_checksum_jobs")
assert remote.checksum_jobs == 100
def _get_remote(self, config, name):
from dvc.remote import Remote
remote = config.get(name, None)
if not remote:
return None
name = Config.SECTION_REMOTE_FMT.format(remote)
sect = self.project.config._config[name]
return Remote(self.project, sect)
def test_remote_without_checksum_jobs(dvc):
set_config_opts(
dvc,
[
('remote "without_checksum_jobs"', "url", "s3://bucket/name"),
("core", "checksum_jobs", "200"),
],
)
remote = Remote(dvc, name="without_checksum_jobs")
assert remote.checksum_jobs == 200
def _get(stage, p, info, cache, metric):
parsed = urlparse(p)
if parsed.scheme == 'remote':
sect = stage.project.config._config[Config.SECTION_REMOTE_FMT.format(parsed.netloc)]
remote = Remote(stage.project, sect)
return OUTS_MAP[remote.scheme](stage, p, info, cache=cache, remote=remote, metric=metric)
for o in OUTS:
if o.supported(p):
return o(stage, p, info, cache=cache, remote=None, metric=metric)
raise DvcException('Output \'{}\' is not supported'.format(p))
def _get(stage, p, info):
parsed = urlparse(p)
if parsed.scheme == 'remote':
sect = stage.project.config._config[Config.SECTION_REMOTE_FMT.format(parsed.netloc)]
remote = Remote(stage.project, sect)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
raise DvcException('Dependency \'{}\' is not supported'.format(p))
def _get(stage, p, info):
parsed = urlparse(p)
if parsed.scheme == 'remote':
name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
sect = stage.project.config.config[name]
remote = Remote(stage.project, sect)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return DependencyLOCAL(stage, p, info)
def _get(stage, p, info):
from dvc.utils.compat import urlparse
parsed = urlparse(p)
if parsed.scheme == "remote":
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return DependencyLOCAL(stage, p, info)
def __init__(self, repo):
from dvc.remote import Remote
self.repo = repo
self.config = config = repo.config["cache"]
local = config.get("local")
if local:
settings = {"name": local}
else:
settings = {**config, "url": config["dir"]}
self.local = Remote(repo, **settings)
def _get(stage, p, info):
parsed = urlparse(p)
if parsed.scheme == "remote":
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
if info and info.get(DependencyREPO.PARAM_REPO):
repo = info.pop(DependencyREPO.PARAM_REPO)
return DependencyREPO(repo, stage, p, info)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return DependencyLOCAL(stage, p, info)
def _get(stage, p, info):
from dvc.utils.compat import urlparse
parsed = urlparse(p)
if parsed.scheme == "remote":
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
if info and info.get(DependencyPKG.PARAM_PKG):
pkg = info.pop(DependencyPKG.PARAM_PKG)
return DependencyPKG(pkg, stage, p, info)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return DependencyLOCAL(stage, p, info)
def _get(stage, p, info):
parsed = urlparse(p) if p else None
if parsed and parsed.scheme == "remote":
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
if info and info.get(RepoDependency.PARAM_REPO):
repo = info.pop(RepoDependency.PARAM_REPO)
return RepoDependency(repo, stage, p, info)
if info and info.get(ParamsDependency.PARAM_PARAMS):
params = info.pop(ParamsDependency.PARAM_PARAMS)
return ParamsDependency(stage, p, params)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return LocalDependency(stage, p, info)
def _get_remote(self, config, name):
"""
The config file is stored in a way that allows you to have a
cache for each remote.
This is needed when specifying external outputs
(as they require you to have an external cache location).
Imagine a config file like the following:
['remote "dvc-storage"']
url = ssh://localhost/tmp
ask_password = true
[cache]
ssh = dvc-storage
This method resolves the name under the cache section into the
correct Remote instance.
Args:
config (dict): The cache section on the config file
name (str): Name of the section we are interested in to retrieve
Returns:
remote (dvc.Remote): Remote instance that the section is referring.
None when there's no remote with that name.
Example:
>>> _get_remote(config={'ssh': 'dvc-storage'}, name='ssh')
"""
from dvc.remote import Remote
remote = config.get(name)
if not remote:
return None
settings = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, settings)
def _get(stage, p, info, cache, metric, persist=False, tags=None):
parsed = urlparse(p)
if parsed.scheme == "remote":
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
return OUTS_MAP[remote.scheme](
stage,
p,
info,
cache=cache,
remote=remote,
metric=metric,
persist=persist,
tags=tags,
)
for o in OUTS:
if o.supported(p):
return o(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
persist=persist,
tags=tags,
)
return OutputLOCAL(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
persist=persist,
tags=tags,
)
def _init_remote(self, name):
return Remote(self.repo, name=name)
def _init_remote(self, remote):
config = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, config)
def test_unsupported(self):
with self.assertRaises(UnsupportedRemoteError):
Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})
def test_remote_without_checksum_jobs_default(dvc):
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
remote = Remote(dvc, name="without_checksum_jobs")
assert remote.checksum_jobs == remote.CHECKSUM_JOBS
