def transfer(self,
source,
odb=None,
jobs=None,
update=False,
no_progress_bar=False):
if odb is None:
odb = self.odb
cls, config, from_info = get_cloud_fs(self.repo, url=source)
from_fs = cls(**config)
# When running import-url --to-remote / add --to-remote/-o ... we
# assume that it is unlikely that the odb will contain majority of the
# hashes, so we transfer everything as is (even if that file might
# already be in the cache) and don't waste an upload to scan the layout
# of the source location. But when doing update --to-remote, there is
# a high probability that the odb might contain some of the hashes, so
# we first calculate all the hashes (but don't transfer anything) and
# then only update the missing cache files.
upload = not (update and from_fs.isdir(from_info))
jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
staging, self.meta, obj = ostage(
odb,
from_info,
from_fs,
"md5",
upload=upload,
no_progress_bar=no_progress_bar,
)
otransfer(
staging,
odb,
{obj.hash_info},
jobs=jobs,
hardlink=False,
shallow=False,
)
self.hash_info = obj.hash_info
return obj
def _commit_granular_dir(self, filter_info):
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
staging, _, save_obj = ostage(
self.odb,
self.fs_path,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
ignore=self.dvcignore,
)
save_obj = save_obj.filter(prefix)
checkout_obj = save_obj.get(self.odb, prefix)
otransfer(
staging,
self.odb,
{save_obj.hash_info} | {oid for _, _, oid in save_obj},
shallow=True,
hardlink=True,
)
return checkout_obj
def commit(self, filter_info=None):
if not self.exists:
raise self.DoesNotExistError(self)
assert self.hash_info
if self.use_cache:
granular = (
self.is_dir_checksum
and filter_info
and filter_info != self.fs_path
)
if granular:
obj = self._commit_granular_dir(filter_info)
else:
staging, _, obj = ostage(
self.odb,
filter_info or self.fs_path,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
ignore=self.dvcignore,
)
otransfer(
staging,
self.odb,
{obj.hash_info},
shallow=False,
hardlink=True,
)
self._checkout(
filter_info or self.fs_path,
self.fs,
obj,
self.odb,
relink=True,
ignore=self.dvcignore,
state=self.repo.state,
prompt=prompt.confirm,
)
self.set_exec()