def upload( self, from_info, to_info, total=None, desc=None, callback=None, no_progress_bar=False, **pbar_args, ): is_file_obj = hasattr(from_info, "read") method = "upload_fobj" if is_file_obj else "put_file" if not hasattr(self, method): raise RemoteActionNotImplemented(method, self.scheme) if to_info.scheme != self.scheme: raise NotImplementedError if not is_file_obj: desc = desc or from_info.name stack = contextlib.ExitStack() if not callback: pbar = ui.progress( desc=desc, disable=no_progress_bar, bytes=True, total=total or -1, **pbar_args, ) stack.enter_context(pbar) callback = pbar.as_callback() if total: callback.set_size(total) with stack: if is_file_obj: wrapped = CallbackIOWrapper(callback.relative_update, from_info) # `size` is used to provide hints to the WebdavFileSystem # for legacy servers. # pylint: disable=no-member return self.upload_fobj(wrapped, to_info, size=total) if from_info.scheme != "local": raise NotImplementedError logger.debug("Uploading '%s' to '%s'", from_info, to_info) # pylint: disable=no-member return self.put_file(os.fspath(from_info), to_info, callback=callback)
def progress_iter(stages: Stages) -> Iterator["Stage"]: total = len(stages) desc = "Adding..." with ui.progress(stages, total=total, desc=desc, unit="file", leave=True) as pbar: if total == 1: pbar.bar_format = desc pbar.refresh() for stage in pbar: if total > 1: pbar.set_msg(f"{stage.outs[0]}") yield stage if total == 1: # restore bar format for stats # pylint: disable=no-member pbar.bar_format = pbar.BAR_FMT_DEFAULT
def add( # noqa: C901 repo: "Repo", targets: "TargetType", recursive: bool = False, no_commit: bool = False, fname: str = None, to_remote: bool = False, **kwargs: Any, ): to_cache = bool(kwargs.get("out")) and not to_remote transfer = to_remote or to_cache glob = kwargs.get("glob", False) add_targets = collect_targets(repo, targets, recursive, glob) # pass one for creating stages, other one is used for iterating here add_targets, sources = tee(add_targets) # collect targets and build stages as we go desc = "Collecting targets" stages_it = create_stages(repo, add_targets, fname, transfer, **kwargs) stages = list(ui.progress(stages_it, desc=desc, unit="file")) msg = "Collecting stages from the workspace" with translate_graph_error(stages), ui.status(msg) as status: # remove existing stages that are to-be replaced with these # new stages for the graph checks. old_stages = set(repo.stages) - set(stages) status.update("Checking graph") repo.check_modified_graph(stages, list(old_stages)) odb = None if to_remote: odb = repo.cloud.get_remote_odb(kwargs.get("remote"), "add") with warn_link_failures() as link_failures: for stage, source in zip(progress_iter(stages), sources): if to_remote or to_cache: stage.transfer(source, to_remote=to_remote, odb=odb, **kwargs) else: try: stage.save() if not no_commit: stage.commit() except CacheLinkError: link_failures.append(str(stage.relpath)) stage.dump() return stages
def download( self, from_info, to_info, name=None, callback=None, no_progress_bar=False, jobs=None, _only_file=False, **kwargs, ): if not hasattr(self, "get_file"): raise RemoteActionNotImplemented("get_file", self.scheme) if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme != "local": raise NotImplementedError download_dir = not _only_file and self.isdir(from_info) desc = name or to_info.name stack = contextlib.ExitStack() if not callback: pbar_kwargs = {"unit": "Files"} if download_dir else {} pbar = ui.progress( total=-1, desc="Downloading directory" if download_dir else desc, bytes=not download_dir, disable=no_progress_bar, **pbar_kwargs, ) stack.enter_context(pbar) callback = pbar.as_callback() with stack: if download_dir: return self._download_dir(from_info, to_info, callback=callback, jobs=jobs, **kwargs) return self._download_file(from_info, to_info, callback=callback)
def download( self, from_info: AnyFSPath, to_info: AnyFSPath, name: str = None, callback=None, no_progress_bar: bool = False, jobs: int = None, _only_file: bool = False, **kwargs: Any, ): from .local import localfs if not hasattr(self, "get_file"): raise RemoteActionNotImplemented("get_file", self.scheme) download_dir = not _only_file and self.isdir(from_info) desc = name or localfs.path.name(to_info) stack = contextlib.ExitStack() if not callback: pbar_kwargs = {"unit": "Files"} if download_dir else {} pbar = ui.progress( total=-1, desc="Downloading directory" if download_dir else desc, bytes=not download_dir, disable=no_progress_bar, **pbar_kwargs, ) stack.enter_context(pbar) callback = pbar.as_callback() with stack: if download_dir: return self._download_dir(from_info, to_info, callback=callback, jobs=jobs, **kwargs) return self._download_file(from_info, to_info, callback=callback)