def cli_mkdir(): """Make directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to create") parser.add_argument( "-p", "--parents", action="store_true", help="no error if existing, make parent directories as needed") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: if args.parents: metadata = client.makedirs(path, exist_ok=True) else: metadata = client.mkdir(path) cprogress("directory '%s' created at '%s'" % (path, metadata["webUrl"])) except Exception as err: cerror("failed to create directory '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_rm(): """Remove CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="PATH", nargs="+", help="path of remote item to remove") parser.add_argument( "-r", "-R", "--recursive", action="store_true", help="remove directories and their contents recursively") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rm(path, recursive=args.recursive) cprogress("'%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_mkdir(): """Make directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to create") parser.add_argument("-p", "--parents", action="store_true", help="no error if existing, make parent directories as needed") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: if args.parents: metadata = client.makedirs(path, exist_ok=True) else: metadata = client.mkdir(path) cprogress("directory '%s' created at '%s'" % (path, metadata["webUrl"])) except Exception as err: cerror("failed to create directory '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def __call__(self, args): """Download a remote file. ``args`` could either be a single path, which is interpreted as path to the remote file to download (to the current working directory), or a pair ``(remotepath, localdir)``, where localdir is interpreted as the destination directory. """ if isinstance(args, tuple): remotepath, localdir = args else: remotepath = args localdir = None try: self._client.download(remotepath, destdir=localdir, **self._download_kwargs) cprogress("finished downloading '%s'" % remotepath) return 0 except KeyboardInterrupt: cerror("download of '%s' interrupted" % remotepath) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to download '%s': %s: %s" % (remotepath, type(err).__name__, str(err))) return 1
def __call__(self, args): """Download a remote file. ``args`` could either be a single path, which is interpreted as path to the remote file to download (to the current working directory), or a pair ``(remotepath, localdir)``, where localdir is interpreted as the destination directory. """ if isinstance(args, tuple): remotepath, localdir = args else: remotepath = args localdir = None try: self._client.download(remotepath, destdir=localdir, **self._download_kwargs) cprogress("finished downloading '%s'" % remotepath) return 0 except KeyboardInterrupt: cerror("download of '%s' interrupted" % remotepath) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to download '%s': %s: %s" % (remotepath, type(err).__name__, str(err))) return 1
def main(): """CLI interface.""" description = "Convert duration in seconds to human readable format." parser = argparse.ArgumentParser(description=description) parser.add_argument("-d", "--decimal-digits", metavar="NUM_DIGITS", nargs="?", type=int, const=2, default=0, help="""Print digits after the decimal point. By default the duration is rounded to whole seconds, but this option enables decimal digits. If NUM_DIGITS argument is given, print that many digits after the decimal point; if this option is specified but no NUM_DIGITS is given, print 2 digits after the decimal point.""") parser.add_argument("-1", "--one-hour-digit", action="store_true", help="""Only print one hour digit when the duration is less than ten hours. By default the hour is zero-padded to two digits.""") parser.add_argument("seconds", type=float, help="Total number of seconds. Must be nonnegative.") args = parser.parse_args() try: print(humantime(args.seconds, ndigits=args.decimal_digits, one_hour_digit=args.one_hour_digit)) except ValueError as err: cerror(str(err)) return 1
def __call__(self, local_path): """Upload a local file.""" try: self._client.upload(self._directory, local_path, **self._upload_kwargs) cprogress("finished uploading '%s'" % local_path) return 0 except KeyboardInterrupt: cerror("upload of '%s' interrupted" % local_path) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to upload '%s' to '%s': %s: %s" % (local_path, self._directory, type(err).__name__, str(err))) return 1
def __call__(self, local_path): """Upload a local file.""" try: self._client.upload(self._directory, local_path, **self._upload_kwargs) cprogress("finished uploading '%s'" % local_path) return 0 except KeyboardInterrupt: cerror("upload of '%s' interrupted" % local_path) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to upload '%s' to '%s': %s: %s" % (local_path, self._directory, type(err).__name__, str(err))) return 1
def _init_client(): """Init a client or exit with 1. Print a helpful error message and exit with code 1 if client initialization somehow fails due to problematic config file. Returns ------- onedrive.api.OneDriveAPIClient """ try: return onedrive.api.OneDriveAPIClient() except OSError as err: cerror(str(err)) exit(1)
def _init_client(): """Init a client or exit with 1. Print a helpful error message and exit with code 1 if client initialization somehow fails due to problematic config file. Returns ------- onedrive.api.OneDriveAPIClient """ try: return onedrive.api.OneDriveAPIClient() except OSError as err: cerror(str(err)) exit(1)
def cli_geturl(): """Get URL CLI.""" parser = argparse.ArgumentParser() parser.add_argument("path", help="remote path (file or directory)") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() path = args.path try: print(client.geturl(path)) return 0 except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) return 1 except Exception as err: cerror("failed to get URL for '%s': %s: %s" % (path, type(err).__name__, str(err))) return 1
def cli_metadata(): """Display metadata CLI.""" parser = argparse.ArgumentParser(description="Dump JSON metadata of item.") parser.add_argument("path", help="remote path (file or directory)") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() path = args.path try: print(json.dumps(client.metadata(path), indent=4)) return 0 except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) return 1 except Exception as err: cerror("failed to get URL for '%s': %s: %s" % (path, type(err).__name__, str(err))) return 1
def cli_rmdir(): """Remove empty directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to remove") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rmdir(path) cprogress("directory '%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_metadata(): """Display metadata CLI.""" parser = argparse.ArgumentParser(description="Dump JSON metadata of item.") parser.add_argument("path", help="remote path (file or directory)") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() path = args.path try: print(json.dumps(client.metadata(path), indent=4)) return 0 except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) return 1 except Exception as err: cerror("failed to get URL for '%s': %s: %s" % (path, type(err).__name__, str(err))) return 1
def cli_geturl(): """Get URL CLI.""" parser = argparse.ArgumentParser() parser.add_argument("path", help="remote path (file or directory)") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() path = args.path try: print(client.geturl(path)) return 0 except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) return 1 except Exception as err: cerror("failed to get URL for '%s': %s: %s" % (path, type(err).__name__, str(err))) return 1
def run(self): """Run the copy operation and monitor status. The exit code is either 0 or 1, indicating success or failure. """ try: if not self._recursive: self._client.assert_file(self.src) self._client.copy(self.src, self.dst, overwrite=self._overwrite, show_progress=self._show_progress) cprogress("finished copying '%s' to '%s'" % (self.src, self.dst)) return 0 except KeyboardInterrupt: cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst)) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to copy '%s' to '%s': %s: %s" % (self.src, self.dst, type(err).__name__, str(err))) return 1
def cli_rm(): """Remove CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="PATH", nargs="+", help="path of remote item to remove") parser.add_argument("-r", "-R", "--recursive", action="store_true", help="remove directories and their contents recursively") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rm(path, recursive=args.recursive) cprogress("'%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_rmdir(): """Remove empty directory CLI.""" parser = argparse.ArgumentParser() parser.add_argument("paths", metavar="DIRECTORY", nargs="+", help="path of remote directory to remove") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() returncode = 0 for path in args.paths: try: client.rmdir(path) cprogress("directory '%s' removed from OneDrive" % path) except Exception as err: cerror("failed to remove '%s': %s: %s" % (path, type(err).__name__, str(err))) returncode = 1 return returncode
def run(self): """Run the copy operation and monitor status. The exit code is either 0 or 1, indicating success or failure. """ try: if not self._recursive: self._client.assert_file(self.src) self._client.copy(self.src, self.dst, overwrite=self._overwrite, show_progress=self._show_progress) cprogress("finished copying '%s' to '%s'" % (self.src, self.dst)) return 0 except KeyboardInterrupt: cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst)) return 1 except Exception as err: # catch any exception in a multiprocessing environment cerror("failed to copy '%s' to '%s': %s: %s" % (self.src, self.dst, type(err).__name__, str(err))) return 1
def cli_mv_or_cp(util, util_name=None): """Mimic the behavior of coreutils ``mv`` or ``cp``. Parameters ---------- util : {"mv", "cp"} util_name : str, optional Utility name shown in usage and help text. If omitted, will be set to the value of ``util``. """ usage = """ {util_name} [options] [-T] SOURCE DEST {util_name} [options] SOURCE... DIRECTORY {util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name) parser = argparse.ArgumentParser(usage=usage) parser.add_argument("paths", nargs="+", help="sources, destination or directory, depending on invocation") parser.add_argument("-t", "--target-directory", action="store_true", help="copy all SOURCE arguments into DIRECTORY") parser.add_argument("-T", "--no-target-directory", action="store_true", help="treat DEST as exact destination, not directory") parser.add_argument("-f", "--force", action="store_true", help="overwrite existing destinations") if util == "cp": parser.add_argument("-R", "-r", "--recursive", action="store_true", help="copy directories recursively") args = parser.parse_args() onedrive.log.logging_setup() client = None # defer setup # list of (src, dst) pairs, where dst is the full destination src_dst_list = [] if args.target_directory and args.no_target_directory: cfatal_error("conflicting options -t and -T; see %s -h" % util_name) return 1 if len(args.paths) < 2: cfatal_error("at least two paths required; see %s -h" % util_name) return 1 elif len(args.paths) == 2: # single source item if args.target_directory: # mv/cp -t DIRECTORY SOURCE directory, source = args.paths dest = posixpath.join(directory, posixpath.basename(source)) elif args.no_target_directory: # mv/cp -T SOURCE DEST source, dest = args.paths else: # no -t or -T flag # automatically decide based on whether dest is an existing directory client = _init_client() if client.isdir(args.paths[1]): # mv/cp SOURCE DIRECTORY source, directory = args.paths dest = posixpath.join(directory, posixpath.basename(source)) else: # mv/cp SOURCE DEST source, dest = args.paths src_dst_list.append((source, dest)) else: # multiple source items if args.no_target_directory: cerror("option -T cannot be specified when there are multiple source items") return 1 elif args.target_directory: # mv/cp -t DIRECTORY SOURCE... sources = args.paths[1:] directory = args.paths[0] else: # mv/cp SOURCE... DIRECTORY sources = args.paths[:-1] directory = args.paths[-1] src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source))) for source in sources] if client is None: client = _init_client() # 3, 2, 1, action! returncode = 0 if util == "mv": # move each item synchronously for src_dst_pair in src_dst_list: src, dst = src_dst_pair try: client.move(src, dst, overwrite=args.force) cprogress("moved '%s' to '%s'" % (src, dst)) except Exception as err: cerror("failed to move '%s' to '%s': %s: %s" % (src, dst, type(err).__name__, str(err))) returncode = 1 else: # cp is more involved num_items = len(src_dst_list) show_progress = (num_items == 1) and zmwangx.pbar.autopbar() workers = [CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force, show_progress=show_progress) for src, dst in src_dst_list] try: for worker in workers: worker.start() for worker in workers: worker.join() if worker.exitcode != 0: returncode = 1 except KeyboardInterrupt: returncode = 1 return returncode
def cli_dirupload(): """Directory upload CLI.""" # TODO: how to handle uploading to an existing and non-empty directory tree? # TODO: concurrency # TODO: option to skip creating directories (useful for resuming dirupload) parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote *parent* directory to upload to") parser.add_argument("localdir", help="path to the local directory to upload") parser.add_argument("-n", "--name", help="""name of the remote directory (by default it is just the basename of the local directory)""") args = parser.parse_args() localroot = os.path.abspath(args.localdir) remoteparent = args.remotedir remotename = args.name if args.name is not None else os.path.basename( localroot) remoteroot = posixpath.join(remoteparent, remotename) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localroot): cfatal_error("'%s' is not an existing local directory" % localroot) return 1 if not client.isdir(remoteparent): cfatal_error("'%s' is not an existing remote directory" % remoteparent) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating directories...") # uploads is a list of tuples (remotedir, localfile, filesize) to upload # TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions # TODO: save calls by creating leaves only (topdown false, and use a # set to keep track of already created relpaths, and once a leaf is # created, add to the set itself and all its parents) uploads = [] for localdir, _, files in os.walk(localroot): normalized_relpath = onedrive.util.normalized_posixpath( os.path.relpath(localdir, start=localroot)) remotedir = posixpath.normpath( posixpath.join(remoteroot, normalized_relpath)) client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok? print(remotedir, file=sys.stderr) for filename in files: localfile = os.path.join(localdir, filename) uploads.append( (remotedir, localfile, os.path.getsize(localfile))) # upload files in ascending order of filesize uploads = sorted(uploads, key=lambda upload: upload[2]) returncode = 0 total = remaining = len(uploads) total_bytes = remaining_bytes = sum([upload[2] for upload in uploads]) cprogress("uploading %d files..." % total) for upload in uploads: remotedir, localfile, filesize = upload cprogress("remaining: %d/%d files, %s/%s" % (remaining, total, zmwangx.humansize.humansize( remaining_bytes, prefix="iec", unit=""), zmwangx.humansize.humansize( total_bytes, prefix="iec", unit=""))) try: client.upload(remotedir, localfile, show_progress=show_progress) cprogress("finished uploading '%s'" % localfile) except Exception as err: cerror("failed to upload '%s' to '%s': %s: %s" % (localfile, remotedir, type(err).__name__, str(err))) returncode = 1 remaining -= 1 remaining_bytes -= filesize return returncode except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_mv_or_cp(util, util_name=None): """Mimic the behavior of coreutils ``mv`` or ``cp``. Parameters ---------- util : {"mv", "cp"} util_name : str, optional Utility name shown in usage and help text. If omitted, will be set to the value of ``util``. """ usage = """ {util_name} [options] [-T] SOURCE DEST {util_name} [options] SOURCE... DIRECTORY {util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name) parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "paths", nargs="+", help="sources, destination or directory, depending on invocation") parser.add_argument("-t", "--target-directory", action="store_true", help="copy all SOURCE arguments into DIRECTORY") parser.add_argument("-T", "--no-target-directory", action="store_true", help="treat DEST as exact destination, not directory") parser.add_argument("-f", "--force", action="store_true", help="overwrite existing destinations") if util == "cp": parser.add_argument("-R", "-r", "--recursive", action="store_true", help="copy directories recursively") args = parser.parse_args() onedrive.log.logging_setup() client = None # defer setup # list of (src, dst) pairs, where dst is the full destination src_dst_list = [] if args.target_directory and args.no_target_directory: cfatal_error("conflicting options -t and -T; see %s -h" % util_name) return 1 if len(args.paths) < 2: cfatal_error("at least two paths required; see %s -h" % util_name) return 1 elif len(args.paths) == 2: # single source item if args.target_directory: # mv/cp -t DIRECTORY SOURCE directory, source = args.paths dest = posixpath.join(directory, posixpath.basename(source)) elif args.no_target_directory: # mv/cp -T SOURCE DEST source, dest = args.paths else: # no -t or -T flag # automatically decide based on whether dest is an existing directory client = _init_client() if client.isdir(args.paths[1]): # mv/cp SOURCE DIRECTORY source, directory = args.paths dest = posixpath.join(directory, posixpath.basename(source)) else: # mv/cp SOURCE DEST source, dest = args.paths src_dst_list.append((source, dest)) else: # multiple source items if args.no_target_directory: cerror( "option -T cannot be specified when there are multiple source items" ) return 1 elif args.target_directory: # mv/cp -t DIRECTORY SOURCE... sources = args.paths[1:] directory = args.paths[0] else: # mv/cp SOURCE... DIRECTORY sources = args.paths[:-1] directory = args.paths[-1] src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source))) for source in sources] if client is None: client = _init_client() # 3, 2, 1, action! returncode = 0 if util == "mv": # move each item synchronously for src_dst_pair in src_dst_list: src, dst = src_dst_pair try: client.move(src, dst, overwrite=args.force) cprogress("moved '%s' to '%s'" % (src, dst)) except Exception as err: cerror("failed to move '%s' to '%s': %s: %s" % (src, dst, type(err).__name__, str(err))) returncode = 1 else: # cp is more involved num_items = len(src_dst_list) show_progress = (num_items == 1) and zmwangx.pbar.autopbar() workers = [ CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force, show_progress=show_progress) for src, dst in src_dst_list ] try: for worker in workers: worker.start() for worker in workers: worker.join() if worker.exitcode != 0: returncode = 1 except KeyboardInterrupt: returncode = 1 return returncode
def cli_dirdownload(): """Directory download CLI.""" parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote directory to download") parser.add_argument("localdir", help="path to the local *parent* directory to download to") parser.add_argument("-j", "--jobs", type=int, default=8, help="number of concurrect downloads, use 0 for unlimited; default is 8") parser.add_argument("--no-check", action="store_true", help="do not compare checksum of remote and local files") parser.add_argument("-f", "--fresh", action="store_true", help="discard any previous failed download") parser.add_argument("--curl", dest="downloader", action="store_const", const="curl", help="use curl to download") parser.add_argument("--wget", dest="downloader", action="store_const", const="wget", help="use wget to download") parser.add_argument("-n", "--name", help="""name of the local directory to create (by default it is just the basename of the remote directory)""") args = parser.parse_args() remoteroot = args.remotedir localparent = os.path.abspath(args.localdir) localname = args.name if args.name is not None else os.path.basename(remoteroot) localroot = os.path.join(localparent, localname) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localparent): cfatal_error("'%s' is not an existing local directory" % localparent) return 1 if not client.isdir(remoteroot): cfatal_error("'%s' is not an existing remote directory" % remoteroot) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating local directories...") # downloads is a list of pairs (remotefile, localdir) to download downloads = [] for remotedir, _, files in client.walk(remoteroot, paths_only=True): normalized_relpath = onedrive.util.normalized_ospath( posixpath.relpath(remotedir, start=remoteroot)) localdir = os.path.normpath(os.path.join(localroot, normalized_relpath)) os.makedirs(localdir, exist_ok=True) print(localdir, file=sys.stderr) for filename in files: remotefile = posixpath.join(remotedir, filename) downloads.append((remotefile, localdir)) num_files = len(downloads) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (num_files == 1) and zmwangx.pbar.autopbar() download_kwargs = { "compare_hash": not args.no_check, "show_progress": show_progress, "resume": not args.fresh, "downloader": args.downloader, } cprogress("downloading %d files..." % num_files) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: downloader = Downloader(client, download_kwargs=download_kwargs) returncodes = [] try: returncodes = pool.map(downloader, downloads, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0 except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_dirdownload(): """Directory download CLI.""" parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote directory to download") parser.add_argument( "localdir", help="path to the local *parent* directory to download to") parser.add_argument( "-j", "--jobs", type=int, default=8, help="number of concurrect downloads, use 0 for unlimited; default is 8" ) parser.add_argument( "--no-check", action="store_true", help="do not compare checksum of remote and local files") parser.add_argument("-f", "--fresh", action="store_true", help="discard any previous failed download") parser.add_argument("--curl", dest="downloader", action="store_const", const="curl", help="use curl to download") parser.add_argument("--wget", dest="downloader", action="store_const", const="wget", help="use wget to download") parser.add_argument("-n", "--name", help="""name of the local directory to create (by default it is just the basename of the remote directory)""") args = parser.parse_args() remoteroot = args.remotedir localparent = os.path.abspath(args.localdir) localname = args.name if args.name is not None else os.path.basename( remoteroot) localroot = os.path.join(localparent, localname) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localparent): cfatal_error("'%s' is not an existing local directory" % localparent) return 1 if not client.isdir(remoteroot): cfatal_error("'%s' is not an existing remote directory" % remoteroot) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating local directories...") # downloads is a list of pairs (remotefile, localdir) to download downloads = [] for remotedir, _, files in client.walk(remoteroot, paths_only=True): normalized_relpath = onedrive.util.normalized_ospath( posixpath.relpath(remotedir, start=remoteroot)) localdir = os.path.normpath( os.path.join(localroot, normalized_relpath)) os.makedirs(localdir, exist_ok=True) print(localdir, file=sys.stderr) for filename in files: remotefile = posixpath.join(remotedir, filename) downloads.append((remotefile, localdir)) num_files = len(downloads) jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files show_progress = (num_files == 1) and zmwangx.pbar.autopbar() download_kwargs = { "compare_hash": not args.no_check, "show_progress": show_progress, "resume": not args.fresh, "downloader": args.downloader, } cprogress("downloading %d files..." % num_files) with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool: downloader = Downloader(client, download_kwargs=download_kwargs) returncodes = [] try: returncodes = pool.map(downloader, downloads, chunksize=1) except KeyboardInterrupt: returncodes.append(1) return 1 if 1 in returncodes else 0 except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_ls(): """List items CLI.""" description = """\ ls for OneDrive. By default the long format is used, i.e., for each entry, the following fields are printed: type, childcount, size, indentation, name. * type is a single character, `d' for a directory and `-' for a file; * childcount is an integer for a directory and `-' for a file; * size is the total size of the item (the size of a directory is calculated recursively); this is by default a human-readable string, and can be switched to plain byte count using the +h, ++human flag; * indentation only appears in tree mode, where each level of depth is translated into a four-space indent; * name is the basename of the item (note this important difference from POSIX ls, which would show full arguments under some circumstances; this is considered a defect and might change in the future). Long format can be turned off using the +l, ++long format, in which case only indentations and names are printed. Note that you can turn on **tree mode** using the -t, --tree flag, in which case full directory trees are printed (on the fly). There is also the -d flag for toggling directory only mode, similar to -d of ls(1) or tree(1) (for tree mode). Please do not rely on the output format of ls to be stable, and especially do not parse it programatically, since there is no guarantee that it won't change in the future. Please use the API instead (children, listdir, walk, walkn, etc.; see the onedrive.api module). """ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(description), prefix_chars="-+") parser.add_argument("paths", metavar="PATH", nargs="+", help="remote path(s)") parser.add_argument("-d", "--directory", action="store_true", help="""list directories themselves, not their contents; when used in conjuction with the --tree option, omit files in a directory tree""") parser.add_argument("-t", "--tree", action="store_true", help="display full directory trees") parser.add_argument("+h", "++human", action="store_false", help="turn off human readable format") parser.add_argument("+l", "++long", action="store_false", help="turn off long format") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() paths = args.paths dironly = args.directory tree = args.tree human = args.human long = args.long # wrap the whole thing into a try block, since ls might take a long # time in tree mode, and the program could be interrupted returncode = 0 try: # categorize paths into files and dirs; files will come first # the files and dirs lists are lists of tuples (path, metadata) files = [] dirs = [] for path in paths: try: metadata = client.metadata(path) if "file" in metadata: files.append((path, metadata)) else: dirs.append((path, metadata)) except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) returncode = 1 # first list files, if any for _, filemetadata in files: _cli_ls_print_entry(filemetadata, long=long, human=human) if not dirs: return returncode # list directories # common kwargs kwargs = {"dironly": dironly, "tree": tree, "human": human, "long": long} # handle first directory specially due to special blank line annoyance firstdirpath, firstdirmetadata = dirs[0] if not dironly or tree: if files: print("") # do not print "dirname:" if we are only listing a single directory if len(files) != 0 or len(dirs) != 1: print("%s:" % firstdirpath) try: _cli_ls_single_directory(client, firstdirpath, metadata=firstdirmetadata, **kwargs) except Exception as err: cerror("failed to list '%s': %s: %s" % (firstdirpath, type(err).__name__, str(err))) for dirpath, dirmetadata in dirs[1:]: if not dironly or tree: print("") print("%s:" % dirpath) try: _cli_ls_single_directory(client, dirpath, metadata=dirmetadata, **kwargs) except Exception as err: cerror("failed to list '%s': %s: %s" % (dirpath, type(err).__name__, str(err))) except KeyboardInterrupt: cerror("interrupted") returncode = 1 return returncode
def cli_rename(): """Batch renaming CLI.""" parser = argparse.ArgumentParser( description="Batch rename all items in a directory.") parser.add_argument("-F", "--files-only", action="store_true", help="""only process files (by default direct subdirectories are also processed)""") parser.add_argument( "-d", "--dry-run", action="store_true", help="""print what would be renamed to stderr but don't do them""") parser.add_argument("-s", "--show", action="store_true", help="""print what was renamed to what""") parser.add_argument("stmts", help="""Python statement(s) to be executed on each filename; the statement(s) should modify the '_' variable, which holds the filename; note that the STL 're' module is already imported for you, and the statement(s) can also make use of a variable 'n', which is the one-based index of the filename currently being handled (indexed in alphabetical order)""") parser.add_argument("directory", help="path to remote directory") args = parser.parse_args() stmts = args.stmts try: ast.parse(stmts) except SyntaxError as err: cfatal_error("invalid statements '%s': %s" % (stmts, str(err))) return 1 onedrive.log.logging_setup() client = _init_client() directory = args.directory try: client.assert_dir(directory) except Exception as err: cfatal_error("%s: %s" % (type(err).__name__, str(err))) return 1 returncode = 0 children = client.children(directory) index = 0 for child in children: if args.files_only and "folder" in child: continue oldname = child["name"] index += 1 globaldict = {"re": re} localdict = {"_": oldname, "n": index} try: # pylint: disable=exec-used exec(stmts, globaldict, localdict) newname = localdict["_"] except Exception as err: cerror("failed to generate new name for '%s': %s: %s" % (oldname, type(err).__name__, str(err))) returncode = 1 continue if oldname == newname: continue if args.dry_run or args.show: print("%s => %s" % (oldname, newname)) if not args.dry_run: try: client.move(posixpath.join(directory, oldname), posixpath.join(directory, newname)) except Exception as err: cerror("failed to move '%s' for '%s': %s: %s" % (oldname, newname, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_rename(): """Batch renaming CLI.""" parser = argparse.ArgumentParser(description="Batch rename all items in a directory.") parser.add_argument("-F", "--files-only", action="store_true", help="""only process files (by default direct subdirectories are also processed)""") parser.add_argument("-d", "--dry-run", action="store_true", help="""print what would be renamed to stderr but don't do them""") parser.add_argument("-s", "--show", action="store_true", help="""print what was renamed to what""") parser.add_argument("stmts", help="""Python statement(s) to be executed on each filename; the statement(s) should modify the '_' variable, which holds the filename; note that the STL 're' module is already imported for you, and the statement(s) can also make use of a variable 'n', which is the one-based index of the filename currently being handled (indexed in alphabetical order)""") parser.add_argument("directory", help="path to remote directory") args = parser.parse_args() stmts = args.stmts try: ast.parse(stmts) except SyntaxError as err: cfatal_error("invalid statements '%s': %s" % (stmts, str(err))) return 1 onedrive.log.logging_setup() client = _init_client() directory = args.directory try: client.assert_dir(directory) except Exception as err: cfatal_error("%s: %s" % (type(err).__name__, str(err))) return 1 returncode = 0 children = client.children(directory) index = 0 for child in children: if args.files_only and "folder" in child: continue oldname = child["name"] index += 1 globaldict = {"re": re} localdict = {"_": oldname, "n": index} try: # pylint: disable=exec-used exec(stmts, globaldict, localdict) newname = localdict["_"] except Exception as err: cerror("failed to generate new name for '%s': %s: %s" % (oldname, type(err).__name__, str(err))) returncode = 1 continue if oldname == newname: continue if args.dry_run or args.show: print("%s => %s" % (oldname, newname)) if not args.dry_run: try: client.move(posixpath.join(directory, oldname), posixpath.join(directory, newname)) except Exception as err: cerror("failed to move '%s' for '%s': %s: %s" % (oldname, newname, type(err).__name__, str(err))) returncode = 1 return returncode
def cli_dirupload(): """Directory upload CLI.""" # TODO: how to handle uploading to an existing and non-empty directory tree? # TODO: concurrency # TODO: option to skip creating directories (useful for resuming dirupload) parser = argparse.ArgumentParser() parser.add_argument("remotedir", help="remote *parent* directory to upload to") parser.add_argument("localdir", help="path to the local directory to upload") parser.add_argument("-n", "--name", help="""name of the remote directory (by default it is just the basename of the local directory)""") args = parser.parse_args() localroot = os.path.abspath(args.localdir) remoteparent = args.remotedir remotename = args.name if args.name is not None else os.path.basename(localroot) remoteroot = posixpath.join(remoteparent, remotename) onedrive.log.logging_setup() client = _init_client() if not os.path.isdir(localroot): cfatal_error("'%s' is not an existing local directory" % localroot) return 1 if not client.isdir(remoteparent): cfatal_error("'%s' is not an existing remote directory" % remoteparent) return 1 try: # KeyboardInterrupt guard block show_progress = zmwangx.pbar.autopbar() cprogress("creating directories...") # uploads is a list of tuples (remotedir, localfile, filesize) to upload # TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions # TODO: save calls by creating leaves only (topdown false, and use a # set to keep track of already created relpaths, and once a leaf is # created, add to the set itself and all its parents) uploads = [] for localdir, _, files in os.walk(localroot): normalized_relpath = onedrive.util.normalized_posixpath( os.path.relpath(localdir, start=localroot)) remotedir = posixpath.normpath(posixpath.join(remoteroot, normalized_relpath)) client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok? print(remotedir, file=sys.stderr) for filename in files: localfile = os.path.join(localdir, filename) uploads.append((remotedir, localfile, os.path.getsize(localfile))) # upload files in ascending order of filesize uploads = sorted(uploads, key=lambda upload: upload[2]) returncode = 0 total = remaining = len(uploads) total_bytes = remaining_bytes = sum([upload[2] for upload in uploads]) cprogress("uploading %d files..." % total) for upload in uploads: remotedir, localfile, filesize = upload cprogress("remaining: %d/%d files, %s/%s" % (remaining, total, zmwangx.humansize.humansize(remaining_bytes, prefix="iec", unit=""), zmwangx.humansize.humansize(total_bytes, prefix="iec", unit=""))) try: client.upload(remotedir, localfile, show_progress=show_progress) cprogress("finished uploading '%s'" % localfile) except Exception as err: cerror("failed to upload '%s' to '%s': %s: %s" % (localfile, remotedir, type(err).__name__, str(err))) returncode = 1 remaining -= 1 remaining_bytes -= filesize return returncode except KeyboardInterrupt: cerror("interrupted" % localfile) return 1
def cli_ls(): """List items CLI.""" description = """\ ls for OneDrive. By default the long format is used, i.e., for each entry, the following fields are printed: type, childcount, size, indentation, name. * type is a single character, `d' for a directory and `-' for a file; * childcount is an integer for a directory and `-' for a file; * size is the total size of the item (the size of a directory is calculated recursively); this is by default a human-readable string, and can be switched to plain byte count using the +h, ++human flag; * indentation only appears in tree mode, where each level of depth is translated into a four-space indent; * name is the basename of the item (note this important difference from POSIX ls, which would show full arguments under some circumstances; this is considered a defect and might change in the future). Long format can be turned off using the +l, ++long format, in which case only indentations and names are printed. Note that you can turn on **tree mode** using the -t, --tree flag, in which case full directory trees are printed (on the fly). There is also the -d flag for toggling directory only mode, similar to -d of ls(1) or tree(1) (for tree mode). Please do not rely on the output format of ls to be stable, and especially do not parse it programatically, since there is no guarantee that it won't change in the future. Please use the API instead (children, listdir, walk, walkn, etc.; see the onedrive.api module). """ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(description), prefix_chars="-+") parser.add_argument("paths", metavar="PATH", nargs="+", help="remote path(s)") parser.add_argument("-d", "--directory", action="store_true", help="""list directories themselves, not their contents; when used in conjuction with the --tree option, omit files in a directory tree""") parser.add_argument("-t", "--tree", action="store_true", help="display full directory trees") parser.add_argument("+h", "++human", action="store_false", help="turn off human readable format") parser.add_argument("+l", "++long", action="store_false", help="turn off long format") args = parser.parse_args() onedrive.log.logging_setup() client = _init_client() paths = args.paths dironly = args.directory tree = args.tree human = args.human long = args.long # wrap the whole thing into a try block, since ls might take a long # time in tree mode, and the program could be interrupted returncode = 0 try: # categorize paths into files and dirs; files will come first # the files and dirs lists are lists of tuples (path, metadata) files = [] dirs = [] for path in paths: try: metadata = client.metadata(path) if "file" in metadata: files.append((path, metadata)) else: dirs.append((path, metadata)) except onedrive.exceptions.FileNotFoundError: cerror("'%s' not found on OneDrive" % path) returncode = 1 # first list files, if any for _, filemetadata in files: _cli_ls_print_entry(filemetadata, long=long, human=human) if not dirs: return returncode # list directories # common kwargs kwargs = { "dironly": dironly, "tree": tree, "human": human, "long": long } # handle first directory specially due to special blank line annoyance firstdirpath, firstdirmetadata = dirs[0] if not dironly or tree: if files: print("") # do not print "dirname:" if we are only listing a single directory if len(files) != 0 or len(dirs) != 1: print("%s:" % firstdirpath) try: _cli_ls_single_directory(client, firstdirpath, metadata=firstdirmetadata, **kwargs) except Exception as err: cerror("failed to list '%s': %s: %s" % (firstdirpath, type(err).__name__, str(err))) for dirpath, dirmetadata in dirs[1:]: if not dironly or tree: print("") print("%s:" % dirpath) try: _cli_ls_single_directory(client, dirpath, metadata=dirmetadata, **kwargs) except Exception as err: cerror("failed to list '%s': %s: %s" % (dirpath, type(err).__name__, str(err))) except KeyboardInterrupt: cerror("interrupted") returncode = 1 return returncode