def transfer(self, source, odb=None, jobs=None, update=False, no_progress_bar=False): if odb is None: odb = self.odb cls, config, from_info = get_cloud_fs(self.repo, url=source) from_fs = cls(**config) # When running import-url --to-remote / add --to-remote/-o ... we # assume that it is unlikely that the odb will contain majority of the # hashes, so we transfer everything as is (even if that file might # already be in the cache) and don't waste an upload to scan the layout # of the source location. But when doing update --to-remote, there is # a high probability that the odb might contain some of the hashes, so # we first calculate all the hashes (but don't transfer anything) and # then only update the missing cache files. upload = not (update and from_fs.isdir(from_info)) jobs = jobs or min((from_fs.jobs, odb.fs.jobs)) staging, self.meta, obj = ostage( odb, from_info, from_fs, "md5", upload=upload, no_progress_bar=no_progress_bar, ) otransfer( staging, odb, {obj.hash_info}, jobs=jobs, hardlink=False, shallow=False, ) self.hash_info = obj.hash_info return obj
def _commit_granular_dir(self, filter_info): prefix = self.fs.path.parts( self.fs.path.relpath(filter_info, self.fs_path) ) staging, _, save_obj = ostage( self.odb, self.fs_path, self.fs, self.odb.fs.PARAM_CHECKSUM, ignore=self.dvcignore, ) save_obj = save_obj.filter(prefix) checkout_obj = save_obj.get(self.odb, prefix) otransfer( staging, self.odb, {save_obj.hash_info} | {oid for _, _, oid in save_obj}, shallow=True, hardlink=True, ) return checkout_obj
def commit(self, filter_info=None): if not self.exists: raise self.DoesNotExistError(self) assert self.hash_info if self.use_cache: granular = ( self.is_dir_checksum and filter_info and filter_info != self.fs_path ) if granular: obj = self._commit_granular_dir(filter_info) else: staging, _, obj = ostage( self.odb, filter_info or self.fs_path, self.fs, self.odb.fs.PARAM_CHECKSUM, ignore=self.dvcignore, ) otransfer( staging, self.odb, {obj.hash_info}, shallow=False, hardlink=True, ) self._checkout( filter_info or self.fs_path, self.fs, obj, self.odb, relink=True, ignore=self.dvcignore, state=self.repo.state, prompt=prompt.confirm, ) self.set_exec()