class DependencyREPO(DependencyLOCAL):
PARAM_REPO = "repo"
def __init__(self, erepo, stage, *args, **kwargs):
self.erepo = ExternalRepo(stage.repo.dvc_dir, **erepo)
super(DependencyLOCAL, self).__init__(stage, *args, **kwargs)
def _parse_path(self, remote, path):
self.erepo.install(self.repo.cache.local.cache_dir)
out_path = os.path.join(self.erepo.repo.root_dir,
urlparse(path).path.lstrip("/"))
out, = self.erepo.repo.find_outs_by_path(out_path)
self.info = copy.copy(out.info)
self._erepo_stage = copy.copy(out.stage.path)
return self.REMOTE.path_cls(out.cache_path)
@property
def is_in_repo(self):
return False
def dumpd(self):
ret = super(DependencyLOCAL, self).dumpd()
ret[self.PARAM_REPO] = self.erepo.dumpd()
return ret
def download(self, to, resume=False):
self.erepo.repo.fetch(self._erepo_stage)
to.info = copy.copy(self.info)
to.checkout()
def _make_repo(repo_url):
if not repo_url or urlparse(repo_url).scheme == "":
yield Repo(repo_url)
else:
tmp_dir = tempfile.mkdtemp("dvc-repo")
try:
ext_repo = ExternalRepo(tmp_dir, url=repo_url)
ext_repo.install()
yield ext_repo.repo
finally:
remove(tmp_dir)
def get(url, path, out=None, rev=None):
out = out or os.path.basename(urlparse(path).path)
# Creating a directory right beside the output to make sure that they
# are on the same filesystem, so we could take the advantage of
# reflink and/or hardlink. Not using tempfile.TemporaryDirectory
# because it will create a symlink to tmpfs, which defeats the purpose
# and won't work with reflink/hardlink.
dpath = os.path.dirname(os.path.abspath(out))
tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
erepo = ExternalRepo(tmp_dir, url=url, rev=rev)
try:
erepo.install()
# Try any links possible to avoid data duplication.
#
# Not using symlink, because we need to remove cache after we are
# done, and to make that work we would have to copy data over
# anyway before removing the cache, so we might just copy it
# right away.
#
# Also, we can't use theoretical "move" link type here, because
# the same cache file might be used a few times in a directory.
erepo.repo.config.set(
Config.SECTION_CACHE,
Config.SECTION_CACHE_TYPE,
"reflink,hardlink,copy",
)
src = os.path.join(erepo.path, urlparse(path).path.lstrip("/"))
o, = erepo.repo.find_outs_by_path(src)
erepo.repo.fetch(o.stage.path)
o.path_info = PathInfo(os.path.abspath(out))
with o.repo.state:
o.checkout()
finally:
erepo.uninstall()
def _make_repo(repo_url, rev=None):
if not repo_url or urlparse(repo_url).scheme == "":
assert rev is None, "Custom revision is not supported for local repo"
yield Repo(repo_url)
else:
tmp_dir = tempfile.mkdtemp("dvc-repo")
ext_repo = ExternalRepo(tmp_dir, url=repo_url, rev=rev)
try:
ext_repo.install()
yield ext_repo.repo
finally:
ext_repo.uninstall()
def __init__(self, erepo, stage, *args, **kwargs):
self.erepo = ExternalRepo(stage.repo.dvc_dir, **erepo)
super(DependencyLOCAL, self).__init__(stage, *args, **kwargs)