def upload(
self,
from_info,
to_info,
total=None,
desc=None,
callback=None,
no_progress_bar=False,
**pbar_args,
):
is_file_obj = hasattr(from_info, "read")
method = "upload_fobj" if is_file_obj else "put_file"
if not hasattr(self, method):
raise RemoteActionNotImplemented(method, self.scheme)
if to_info.scheme != self.scheme:
raise NotImplementedError
if not is_file_obj:
desc = desc or from_info.name
stack = contextlib.ExitStack()
if not callback:
pbar = ui.progress(
desc=desc,
disable=no_progress_bar,
bytes=True,
total=total or -1,
**pbar_args,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
if total:
callback.set_size(total)
with stack:
if is_file_obj:
wrapped = CallbackIOWrapper(callback.relative_update,
from_info)
# `size` is used to provide hints to the WebdavFileSystem
# for legacy servers.
# pylint: disable=no-member
return self.upload_fobj(wrapped, to_info, size=total)
if from_info.scheme != "local":
raise NotImplementedError
logger.debug("Uploading '%s' to '%s'", from_info, to_info)
# pylint: disable=no-member
return self.put_file(os.fspath(from_info),
to_info,
callback=callback)
def progress_iter(stages: Stages) -> Iterator["Stage"]:
total = len(stages)
desc = "Adding..."
with ui.progress(stages, total=total, desc=desc, unit="file",
leave=True) as pbar:
if total == 1:
pbar.bar_format = desc
pbar.refresh()
for stage in pbar:
if total > 1:
pbar.set_msg(f"{stage.outs[0]}")
yield stage
if total == 1: # restore bar format for stats
# pylint: disable=no-member
pbar.bar_format = pbar.BAR_FMT_DEFAULT
def add( # noqa: C901
repo: "Repo",
targets: "TargetType",
recursive: bool = False,
no_commit: bool = False,
fname: str = None,
to_remote: bool = False,
**kwargs: Any,
):
to_cache = bool(kwargs.get("out")) and not to_remote
transfer = to_remote or to_cache
glob = kwargs.get("glob", False)
add_targets = collect_targets(repo, targets, recursive, glob)
# pass one for creating stages, other one is used for iterating here
add_targets, sources = tee(add_targets)
# collect targets and build stages as we go
desc = "Collecting targets"
stages_it = create_stages(repo, add_targets, fname, transfer, **kwargs)
stages = list(ui.progress(stages_it, desc=desc, unit="file"))
msg = "Collecting stages from the workspace"
with translate_graph_error(stages), ui.status(msg) as status:
# remove existing stages that are to-be replaced with these
# new stages for the graph checks.
old_stages = set(repo.stages) - set(stages)
status.update("Checking graph")
repo.check_modified_graph(stages, list(old_stages))
odb = None
if to_remote:
odb = repo.cloud.get_remote_odb(kwargs.get("remote"), "add")
with warn_link_failures() as link_failures:
for stage, source in zip(progress_iter(stages), sources):
if to_remote or to_cache:
stage.transfer(source, to_remote=to_remote, odb=odb, **kwargs)
else:
try:
stage.save()
if not no_commit:
stage.commit()
except CacheLinkError:
link_failures.append(str(stage.relpath))
stage.dump()
return stages
def download(
self,
from_info,
to_info,
name=None,
callback=None,
no_progress_bar=False,
jobs=None,
_only_file=False,
**kwargs,
):
if not hasattr(self, "get_file"):
raise RemoteActionNotImplemented("get_file", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
download_dir = not _only_file and self.isdir(from_info)
desc = name or to_info.name
stack = contextlib.ExitStack()
if not callback:
pbar_kwargs = {"unit": "Files"} if download_dir else {}
pbar = ui.progress(
total=-1,
desc="Downloading directory" if download_dir else desc,
bytes=not download_dir,
disable=no_progress_bar,
**pbar_kwargs,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
with stack:
if download_dir:
return self._download_dir(from_info,
to_info,
callback=callback,
jobs=jobs,
**kwargs)
return self._download_file(from_info, to_info, callback=callback)
def download(
self,
from_info: AnyFSPath,
to_info: AnyFSPath,
name: str = None,
callback=None,
no_progress_bar: bool = False,
jobs: int = None,
_only_file: bool = False,
**kwargs: Any,
):
from .local import localfs
if not hasattr(self, "get_file"):
raise RemoteActionNotImplemented("get_file", self.scheme)
download_dir = not _only_file and self.isdir(from_info)
desc = name or localfs.path.name(to_info)
stack = contextlib.ExitStack()
if not callback:
pbar_kwargs = {"unit": "Files"} if download_dir else {}
pbar = ui.progress(
total=-1,
desc="Downloading directory" if download_dir else desc,
bytes=not download_dir,
disable=no_progress_bar,
**pbar_kwargs,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
with stack:
if download_dir:
return self._download_dir(from_info,
to_info,
callback=callback,
jobs=jobs,
**kwargs)
return self._download_file(from_info, to_info, callback=callback)