def cli_mkdir(): """Make directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to create") parser.add_argument("-p", "--parents", action="store_true", help="no error if existing, make parent directories as needed") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: if args.parents: metadata = client.makedirs(path, exist_ok=True) else: metadata = client.mkdir(path) cprogress("directory '%s' created at '%s'" % (path, metadata["webUrl"])) except Exception as err: cerror("failed to create directory '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_mkdir(): """Make directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to create") parser.add_argument( "-p", "--parents", action="store_true", help="no error if existing, make parent directories as needed") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: if args.parents: metadata = client.makedirs(path, exist_ok=True) else: metadata = client.mkdir(path) cprogress("directory '%s' created at '%s'" % (path, metadata["webUrl"])) except Exception as err: cerror("failed to create directory '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_rm(): """Remove CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="PATH", nargs="+", help="path of remote item to remove") parser.add_argument( "-r", "-R", "--recursive", action="store_true", help="remove directories and their contents recursively") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rm(path, recursive=args.recursive) cprogress("'%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def __call__(self, args): """Download a remote file. ``args`` could either be a single path, which is interpreted as path to the remote file to download (to the current working directory), or a pair ``(remotepath, localdir)``, where localdir is interpreted as the destination directory. """ if isinstance(args, tuple): remotepath, localdir = args else: remotepath = args localdir = None try: self._client.download(remotepath, destdir=localdir, **self._download_kwargs) cprogress("finished downloading '%s'" % remotepath) return 0 except KeyboardInterrupt: cerror("download of '%s' interrupted" % remotepath) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to download '%s': %s: %s" % (remotepath, type(err).__name__, str(err))) return 1
def __call__(self, args): """Download a remote file. ``args`` could either be a single path, which is interpreted as path to the remote file to download (to the current working directory), or a pair ``(remotepath, localdir)``, where localdir is interpreted as the destination directory. """ if isinstance(args, tuple): remotepath, localdir = args else: remotepath = args localdir = None try: self._client.download(remotepath, destdir=localdir, **self._download_kwargs) cprogress("finished downloading '%s'" % remotepath) return 0 except KeyboardInterrupt: cerror("download of '%s' interrupted" % remotepath) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to download '%s': %s: %s" % (remotepath, type(err).__name__, str(err))) return 1
def authorize_client(self): """Authorize the client using the code flow.""" # get authorization code # # authorization url: # https://login.live.com/oauth20_authorize.srf?client_id={client_id}&scope={scope} # &response_type=code&redirect_uri={redirect_uri} auth_url = urllib.parse.urlunparse(( "https", # scheme "login.live.com", # netloc "oauth20_authorize.srf", # path "", # params urllib.parse.urlencode({ "client_id": self._client_id, "scope": "wl.signin wl.offline_access onedrive.readwrite", "response_type": "code", "redirect_uri": self._redirect_uri, }), # query "", # fragment )) webbrowser.open(auth_url) info = ("You are being directed to your default web browser for authorization. " " When done, copy the URL you are redirected to and paste it back here.") prompt = "Please enter the redirect URL: " try: redirect_url = cprompt(info=info, prompt=prompt) except EOFError: raise EOFError("no input for the redirect URL prompt") code = urllib.parse.parse_qs(urllib.parse.urlparse(redirect_url).query)["code"][0] # redeem the code payload = { "client_id": self._client_id, "redirect_uri": self._redirect_uri, "client_secret": self._client_secret, "code": code, "grant_type": "authorization_code", } headers = {"Content-Type": "application/x-www-form-urlencoded"} redeem_request = requests.post("https://login.live.com/oauth20_token.srf", data=payload, headers=headers) response_json = redeem_request.json() if redeem_request.status_code == 200 and "access_token" in response_json: self._access_token = redeem_request.json()["access_token"] self._refresh_token = redeem_request.json()["refresh_token"] else: raise onedrive.exceptions.APIRequestError(response=redeem_request, request_desc="redeeming request") # rewrite config file self._conf["oauth"]["refresh_token"] = self._refresh_token self._conf.rewrite_configs() cprogress("Refresh token generated and written.")
def __call__(self, local_path): """Upload a local file.""" try: self._client.upload(self._directory, local_path, **self._upload_kwargs) cprogress("finished uploading '%s'" % local_path) return 0 except KeyboardInterrupt: cerror("upload of '%s' interrupted" % local_path) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to upload '%s' to '%s': %s: %s" % (local_path, self._directory, type(err).__name__, str(err))) return 1
def __call__(self, local_path): """Upload a local file.""" try: self._client.upload(self._directory, local_path, **self._upload_kwargs) cprogress("finished uploading '%s'" % local_path) return 0 except KeyboardInterrupt: cerror("upload of '%s' interrupted" % local_path) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to upload '%s' to '%s': %s: %s" % (local_path, self._directory, type(err).__name__, str(err))) return 1
def cli_rmdir(): """Remove empty directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to remove") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rmdir(path) cprogress("directory '%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def run(self): """Run the copy operation and monitor status. The exit code is either 0 or 1, indicating success or failure. """ try: if not self._recursive: self._client.assert_file(self.src) self._client.copy(self.src, self.dst, overwrite=self._overwrite, show_progress=self._show_progress) cprogress("finished copying '%s' to '%s'" % (self.src, self.dst)) return 0 except KeyboardInterrupt: cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst)) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to copy '%s' to '%s': %s: %s" % (self.src, self.dst, type(err).__name__, str(err))) return 1
def cli_rmdir(): """Remove empty directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to remove") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rmdir(path) cprogress("directory '%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_rm(): """Remove CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="PATH", nargs="+", help="path of remote item to remove") parser.add_argument("-r", "-R", "--recursive", action="store_true", help="remove directories and their contents recursively") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rm(path, recursive=args.recursive) cprogress("'%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def run(self): """Run the copy operation and monitor status. The exit code is either 0 or 1, indicating success or failure. """ try: if not self._recursive: self._client.assert_file(self.src) self._client.copy(self.src, self.dst, overwrite=self._overwrite, show_progress=self._show_progress) cprogress("finished copying '%s' to '%s'" % (self.src, self.dst)) return 0 except KeyboardInterrupt: cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst)) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to copy '%s' to '%s': %s: %s" % (self.src, self.dst, type(err).__name__, str(err))) return 1
def cli_dirdownload(): """Directory download CLI.""" parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote directory to download") parser.add_argument( "localdir", help="path to the local *parent* directory to download to") parser.add_argument( "-j", "--jobs", type=int, default=8, help="number of concurrect downloads, use 0 for unlimited; default is 8" ) parser.add_argument( "--no-check", action="store_true", help="do not compare checksum of remote and local files") parser.add_argument("-f", "--fresh", action="store_true", help="discard any previous failed download") parser.add_argument("--curl", dest="downloader", action="store_const", const="curl", help="use curl to download") parser.add_argument("--wget", dest="downloader", action="store_const", const="wget", help="use wget to download") parser.add_argument("-n", "--name", help="""name of the local directory to create (by default it is just the basename of the remote directory)""") args = parser.parse_args() remoteroot = args.remotedir localparent = os.path.abspath(args.localdir) localname = args.name if args.name is not None else os.path.basename( remoteroot) localroot = os.path.join(localparent, localname) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localparent): cfatal_error("'%s' is not an existing local directory" % localparent) return 1 if not client.isdir(remoteroot): cfatal_error("'%s' is not an existing remote directory" % remoteroot) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating local directories...") # downloads is a list of pairs (remotefile, localdir) to download downloads = [] for remotedir, _, files in client.walk(remoteroot, paths_only=True): normalized_relpath = onedrive.util.normalized_ospath( posixpath.relpath(remotedir, start=remoteroot)) localdir = os.path.normpath( os.path.join(localroot, normalized_relpath)) os.makedirs(localdir, exist_ok=True) print(localdir, file=sys.stderr) for filename in files: remotefile = posixpath.join(remotedir, filename) downloads.append((remotefile, localdir)) num_files = len(downloads) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (num_files == 1) and zmwangx.pbar.autopbar() download_kwargs = { "compare_hash": not args.no_check, "show_progress": show_progress, "resume": not args.fresh, "downloader": args.downloader, } cprogress("downloading %d files..." % num_files) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: downloader = Downloader(client, download_kwargs=download_kwargs) returncodes = [] try: returncodes = pool.map(downloader, downloads, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0 except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_upload(): """Upload CLI.""" parser = argparse.ArgumentParser() parser.add_argument("directory", help="remote directory to upload to") parser.add_argument("local_paths", metavar="PATH", nargs="+", help="path(s) of local file(s) to upload") parser.add_argument("-j", "--jobs", type=int, default=4, help="""number of concurrect uploads (i.e., workers), use 0 for unlimited; default is 4""") parser.add_argument("-f", "--force", action="store_true", help="overwrite if the remote file already exists") parser.add_argument("-c", "--chunk-size", type=int, default=10485760, help="""Size in bytes of each chunk in resumable upload; default is 10 MiB, and the chunk size should not exceed 60 MiB""") parser.add_argument("--base-segment-timeout", type=float, default=14, help="""base timeout for uploading a single segment (10MiB), with one second added to this base timeout for each worker; default is 14""") parser.add_argument("--stream", action="store_true", help="""Use streaming workers (that stream each chunk) instead of regular workers; only use this if you are running a great number of workers concurrently, or if you are extremely concerned about memory usage""") parser.add_argument("--simple-upload-threshold", type=int, default=1048576, help="""file size threshold (in bytes) for using chunked, resumable upload API instead of simple, one shot API (less overhead, good for uploading a great number of small files); default is 1 MiB, and the threshold should not exceed 100 MiB""") parser.add_argument("--no-check", action="store_true", help="""do not compare checksum of local and remote files (this prevents you from resuming an upload in case of a failure)""") args = parser.parse_args() num_files = len(args.local_paths) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (jobs == 1) and zmwangx.pbar.autopbar() upload_kwargs = { "conflict_behavior": "replace" if args.force else "fail", "simple_upload_threshold": args.simple_upload_threshold, "compare_hash": not args.no_check, "chunk_size": args.chunk_size, "timeout": args.base_segment_timeout + jobs, "stream": args.stream, "show_progress": show_progress, } onedrive.log.logging_setup() client = _init_client() # check existence of remote directory directory = args.directory try: directory_url = client.geturl(directory) except onedrive.exceptions.GeneralAPIException as err: cfatal_error(str(err)) return 1 if show_progress: cprogress("preparing to upload to '%s'" % directory) cprogress("directory URL: %s" % directory_url) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: uploader = Uploader(client, directory, upload_kwargs) returncodes = [] try: returncodes = pool.map(uploader, args.local_paths, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0
def cli_dirdownload(): """Directory download CLI.""" parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote directory to download") parser.add_argument("localdir", help="path to the local *parent* directory to download to") parser.add_argument("-j", "--jobs", type=int, default=8, help="number of concurrect downloads, use 0 for unlimited; default is 8") parser.add_argument("--no-check", action="store_true", help="do not compare checksum of remote and local files") parser.add_argument("-f", "--fresh", action="store_true", help="discard any previous failed download") parser.add_argument("--curl", dest="downloader", action="store_const", const="curl", help="use curl to download") parser.add_argument("--wget", dest="downloader", action="store_const", const="wget", help="use wget to download") parser.add_argument("-n", "--name", help="""name of the local directory to create (by default it is just the basename of the remote directory)""") args = parser.parse_args() remoteroot = args.remotedir localparent = os.path.abspath(args.localdir) localname = args.name if args.name is not None else os.path.basename(remoteroot) localroot = os.path.join(localparent, localname) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localparent): cfatal_error("'%s' is not an existing local directory" % localparent) return 1 if not client.isdir(remoteroot): cfatal_error("'%s' is not an existing remote directory" % remoteroot) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating local directories...") # downloads is a list of pairs (remotefile, localdir) to download downloads = [] for remotedir, _, files in client.walk(remoteroot, paths_only=True): normalized_relpath = onedrive.util.normalized_ospath( posixpath.relpath(remotedir, start=remoteroot)) localdir = os.path.normpath(os.path.join(localroot, normalized_relpath)) os.makedirs(localdir, exist_ok=True) print(localdir, file=sys.stderr) for filename in files: remotefile = posixpath.join(remotedir, filename) downloads.append((remotefile, localdir)) num_files = len(downloads) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (num_files == 1) and zmwangx.pbar.autopbar() download_kwargs = { "compare_hash": not args.no_check, "show_progress": show_progress, "resume": not args.fresh, "downloader": args.downloader, } cprogress("downloading %d files..." % num_files) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: downloader = Downloader(client, download_kwargs=download_kwargs) returncodes = [] try: returncodes = pool.map(downloader, downloads, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0 except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_mv_or_cp(util, util_name=None): """Mimic the behavior of coreutils ``mv`` or ``cp``. Parameters ---------- util : {"mv", "cp"} util_name : str, optional Utility name shown in usage and help text. If omitted, will be set to the value of ``util``. """ usage = """ {util_name} [options] [-T] SOURCE DEST {util_name} [options] SOURCE... DIRECTORY {util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name) parser = argparse.ArgumentParser(usage=usage) parser.add_argument("paths", nargs="+", help="sources, destination or directory, depending on invocation") parser.add_argument("-t", "--target-directory", action="store_true", help="copy all SOURCE arguments into DIRECTORY") parser.add_argument("-T", "--no-target-directory", action="store_true", help="treat DEST as exact destination, not directory") parser.add_argument("-f", "--force", action="store_true", help="overwrite existing destinations") if util == "cp": parser.add_argument("-R", "-r", "--recursive", action="store_true", help="copy directories recursively") args = parser.parse_args() onedrive.log.logging_setup() client = None # defer setup # list of (src, dst) pairs, where dst is the full destination src_dst_list = [] if args.target_directory and args.no_target_directory: cfatal_error("conflicting options -t and -T; see %s -h" % util_name) return 1 if len(args.paths) < 2: cfatal_error("at least two paths required; see %s -h" % util_name) return 1 elif len(args.paths) == 2: # single source item if args.target_directory: # mv/cp -t DIRECTORY SOURCE directory, source = args.paths dest = posixpath.join(directory, posixpath.basename(source)) elif args.no_target_directory: # mv/cp -T SOURCE DEST source, dest = args.paths else: # no -t or -T flag # automatically decide based on whether dest is an existing directory client = _init_client() if client.isdir(args.paths[1]): # mv/cp SOURCE DIRECTORY source, directory = args.paths dest = posixpath.join(directory, posixpath.basename(source)) else: # mv/cp SOURCE DEST source, dest = args.paths src_dst_list.append((source, dest)) else: # multiple source items if args.no_target_directory: cerror("option -T cannot be specified when there are multiple source items") return 1 elif args.target_directory: # mv/cp -t DIRECTORY SOURCE... sources = args.paths[1:] directory = args.paths[0] else: # mv/cp SOURCE... DIRECTORY sources = args.paths[:-1] directory = args.paths[-1] src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source))) for source in sources] if client is None: client = _init_client() # 3, 2, 1, action! returncode = 0 if util == "mv": # move each item synchronously for src_dst_pair in src_dst_list: src, dst = src_dst_pair try: client.move(src, dst, overwrite=args.force) cprogress("moved '%s' to '%s'" % (src, dst)) except Exception as err: cerror("failed to move '%s' to '%s': %s: %s" % (src, dst, type(err).__name__, str(err))) returncode = 1 else: # cp is more involved num_items = len(src_dst_list) show_progress = (num_items == 1) and zmwangx.pbar.autopbar() workers = [CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force, show_progress=show_progress) for src, dst in src_dst_list] try: for worker in workers: worker.start() for worker in workers: worker.join() if worker.exitcode != 0: returncode = 1 except KeyboardInterrupt: returncode = 1 return returncode
def cli_dirupload(): """Directory upload CLI.""" # TODO: how to handle uploading to an existing and non-empty directory tree? # TODO: concurrency # TODO: option to skip creating directories (useful for resuming dirupload) parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote *parent* directory to upload to") parser.add_argument("localdir", help="path to the local directory to upload") parser.add_argument("-n", "--name", help="""name of the remote directory (by default it is just the basename of the local directory)""") args = parser.parse_args() localroot = os.path.abspath(args.localdir) remoteparent = args.remotedir remotename = args.name if args.name is not None else os.path.basename(localroot) remoteroot = posixpath.join(remoteparent, remotename) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localroot): cfatal_error("'%s' is not an existing local directory" % localroot) return 1 if not client.isdir(remoteparent): cfatal_error("'%s' is not an existing remote directory" % remoteparent) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating directories...") # uploads is a list of tuples (remotedir, localfile, filesize) to upload # TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions # TODO: save calls by creating leaves only (topdown false, and use a # set to keep track of already created relpaths, and once a leaf is # created, add to the set itself and all its parents) uploads = [] for localdir, _, files in os.walk(localroot): normalized_relpath = onedrive.util.normalized_posixpath( os.path.relpath(localdir, start=localroot)) remotedir = posixpath.normpath(posixpath.join(remoteroot, normalized_relpath)) client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok? print(remotedir, file=sys.stderr) for filename in files: localfile = os.path.join(localdir, filename) uploads.append((remotedir, localfile, os.path.getsize(localfile))) # upload files in ascending order of filesize uploads = sorted(uploads, key=lambda upload: upload[2]) returncode = 0 total = remaining = len(uploads) total_bytes = remaining_bytes = sum([upload[2] for upload in uploads]) cprogress("uploading %d files..." % total) for upload in uploads: remotedir, localfile, filesize = upload cprogress("remaining: %d/%d files, %s/%s" % (remaining, total, zmwangx.humansize.humansize(remaining_bytes, prefix="iec", unit=""), zmwangx.humansize.humansize(total_bytes, prefix="iec", unit=""))) try: client.upload(remotedir, localfile, show_progress=show_progress) cprogress("finished uploading '%s'" % localfile) except Exception as err: cerror("failed to upload '%s' to '%s': %s: %s" % (localfile, remotedir, type(err).__name__, str(err))) returncode = 1 remaining -= 1 remaining_bytes -= filesize return returncode except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_dirupload(): """Directory upload CLI.""" # TODO: how to handle uploading to an existing and non-empty directory tree? # TODO: concurrency # TODO: option to skip creating directories (useful for resuming dirupload) parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote *parent* directory to upload to") parser.add_argument("localdir", help="path to the local directory to upload") parser.add_argument("-n", "--name", help="""name of the remote directory (by default it is just the basename of the local directory)""") args = parser.parse_args() localroot = os.path.abspath(args.localdir) remoteparent = args.remotedir remotename = args.name if args.name is not None else os.path.basename( localroot) remoteroot = posixpath.join(remoteparent, remotename) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localroot): cfatal_error("'%s' is not an existing local directory" % localroot) return 1 if not client.isdir(remoteparent): cfatal_error("'%s' is not an existing remote directory" % remoteparent) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating directories...") # uploads is a list of tuples (remotedir, localfile, filesize) to upload # TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions # TODO: save calls by creating leaves only (topdown false, and use a # set to keep track of already created relpaths, and once a leaf is # created, add to the set itself and all its parents) uploads = [] for localdir, _, files in os.walk(localroot): normalized_relpath = onedrive.util.normalized_posixpath( os.path.relpath(localdir, start=localroot)) remotedir = posixpath.normpath( posixpath.join(remoteroot, normalized_relpath)) client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok? print(remotedir, file=sys.stderr) for filename in files: localfile = os.path.join(localdir, filename) uploads.append( (remotedir, localfile, os.path.getsize(localfile))) # upload files in ascending order of filesize uploads = sorted(uploads, key=lambda upload: upload[2]) returncode = 0 total = remaining = len(uploads) total_bytes = remaining_bytes = sum([upload[2] for upload in uploads]) cprogress("uploading %d files..." % total) for upload in uploads: remotedir, localfile, filesize = upload cprogress("remaining: %d/%d files, %s/%s" % (remaining, total, zmwangx.humansize.humansize( remaining_bytes, prefix="iec", unit=""), zmwangx.humansize.humansize( total_bytes, prefix="iec", unit=""))) try: client.upload(remotedir, localfile, show_progress=show_progress) cprogress("finished uploading '%s'" % localfile) except Exception as err: cerror("failed to upload '%s' to '%s': %s: %s" % (localfile, remotedir, type(err).__name__, str(err))) returncode = 1 remaining -= 1 remaining_bytes -= filesize return returncode except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_upload(): """Upload CLI.""" parser = argparse.ArgumentParser() parser.add_argument("directory", help="remote directory to upload to") parser.add_argument("local_paths", metavar="PATH", nargs="+", help="path(s) of local file(s) to upload") parser.add_argument("-j", "--jobs", type=int, default=4, help="""number of concurrect uploads (i.e., workers), use 0 for unlimited; default is 4""") parser.add_argument("-f", "--force", action="store_true", help="overwrite if the remote file already exists") parser.add_argument("-c", "--chunk-size", type=int, default=10485760, help="""Size in bytes of each chunk in resumable upload; default is 10 MiB, and the chunk size should not exceed 60 MiB""") parser.add_argument("--base-segment-timeout", type=float, default=14, help="""base timeout for uploading a single segment (10MiB), with one second added to this base timeout for each worker; default is 14""") parser.add_argument("--stream", action="store_true", help="""Use streaming workers (that stream each chunk) instead of regular workers; only use this if you are running a great number of workers concurrently, or if you are extremely concerned about memory usage""") parser.add_argument("--simple-upload-threshold", type=int, default=1048576, help="""file size threshold (in bytes) for using chunked, resumable upload API instead of simple, one shot API (less overhead, good for uploading a great number of small files); default is 1 MiB, and the threshold should not exceed 100 MiB""") parser.add_argument("--no-check", action="store_true", help="""do not compare checksum of local and remote files (this prevents you from resuming an upload in case of a failure)""") args = parser.parse_args() num_files = len(args.local_paths) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (jobs == 1) and zmwangx.pbar.autopbar() upload_kwargs = { "conflict_behavior": "replace" if args.force else "fail", "simple_upload_threshold": args.simple_upload_threshold, "compare_hash": not args.no_check, "chunk_size": args.chunk_size, "timeout": args.base_segment_timeout + jobs, "stream": args.stream, "show_progress": show_progress, } onedrive.log.logging_setup() client = _init_client() # check existence of remote directory directory = args.directory try: directory_url = client.geturl(directory) except onedrive.exceptions.GeneralAPIException as err: cfatal_error(str(err)) return 1 if show_progress: cprogress("preparing to upload to '%s'" % directory) cprogress("directory URL: %s" % directory_url) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: uploader = Uploader(client, directory, upload_kwargs) returncodes = [] try: returncodes = pool.map(uploader, args.local_paths, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0
def cli_mv_or_cp(util, util_name=None): """Mimic the behavior of coreutils ``mv`` or ``cp``. Parameters ---------- util : {"mv", "cp"} util_name : str, optional Utility name shown in usage and help text. If omitted, will be set to the value of ``util``. """ usage = """ {util_name} [options] [-T] SOURCE DEST {util_name} [options] SOURCE... DIRECTORY {util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name) parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "paths", nargs="+", help="sources, destination or directory, depending on invocation") parser.add_argument("-t", "--target-directory", action="store_true", help="copy all SOURCE arguments into DIRECTORY") parser.add_argument("-T", "--no-target-directory", action="store_true", help="treat DEST as exact destination, not directory") parser.add_argument("-f", "--force", action="store_true", help="overwrite existing destinations") if util == "cp": parser.add_argument("-R", "-r", "--recursive", action="store_true", help="copy directories recursively") args = parser.parse_args() onedrive.log.logging_setup() client = None # defer setup # list of (src, dst) pairs, where dst is the full destination src_dst_list = [] if args.target_directory and args.no_target_directory: cfatal_error("conflicting options -t and -T; see %s -h" % util_name) return 1 if len(args.paths) < 2: cfatal_error("at least two paths required; see %s -h" % util_name) return 1 elif len(args.paths) == 2: # single source item if args.target_directory: # mv/cp -t DIRECTORY SOURCE directory, source = args.paths dest = posixpath.join(directory, posixpath.basename(source)) elif args.no_target_directory: # mv/cp -T SOURCE DEST source, dest = args.paths else: # no -t or -T flag # automatically decide based on whether dest is an existing directory client = _init_client() if client.isdir(args.paths[1]): # mv/cp SOURCE DIRECTORY source, directory = args.paths dest = posixpath.join(directory, posixpath.basename(source)) else: # mv/cp SOURCE DEST source, dest = args.paths src_dst_list.append((source, dest)) else: # multiple source items if args.no_target_directory: cerror( "option -T cannot be specified when there are multiple source items" ) return 1 elif args.target_directory: # mv/cp -t DIRECTORY SOURCE... sources = args.paths[1:] directory = args.paths[0] else: # mv/cp SOURCE... DIRECTORY sources = args.paths[:-1] directory = args.paths[-1] src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source))) for source in sources] if client is None: client = _init_client() # 3, 2, 1, action! returncode = 0 if util == "mv": # move each item synchronously for src_dst_pair in src_dst_list: src, dst = src_dst_pair try: client.move(src, dst, overwrite=args.force) cprogress("moved '%s' to '%s'" % (src, dst)) except Exception as err: cerror("failed to move '%s' to '%s': %s: %s" % (src, dst, type(err).__name__, str(err))) returncode = 1 else: # cp is more involved num_items = len(src_dst_list) show_progress = (num_items == 1) and zmwangx.pbar.autopbar() workers = [ CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force, show_progress=show_progress) for src, dst in src_dst_list ] try: for worker in workers: worker.start() for worker in workers: worker.join() if worker.exitcode != 0: returncode = 1 except KeyboardInterrupt: returncode = 1 return returncode