def cli_mkdir():
"""Make directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="DIRECTORY",
nargs="+",
help="path of remote directory to create")
parser.add_argument(
"-p",
"--parents",
action="store_true",
help="no error if existing, make parent directories as needed")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
if args.parents:
metadata = client.makedirs(path, exist_ok=True)
else:
metadata = client.mkdir(path)
cprogress("directory '%s' created at '%s'" %
(path, metadata["webUrl"]))
except Exception as err:
cerror("failed to create directory '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_rm():
"""Remove CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="PATH",
nargs="+",
help="path of remote item to remove")
parser.add_argument(
"-r",
"-R",
"--recursive",
action="store_true",
help="remove directories and their contents recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rm(path, recursive=args.recursive)
cprogress("'%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_mkdir():
"""Make directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="DIRECTORY", nargs="+",
help="path of remote directory to create")
parser.add_argument("-p", "--parents", action="store_true",
help="no error if existing, make parent directories as needed")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
if args.parents:
metadata = client.makedirs(path, exist_ok=True)
else:
metadata = client.mkdir(path)
cprogress("directory '%s' created at '%s'" %
(path, metadata["webUrl"]))
except Exception as err:
cerror("failed to create directory '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def __call__(self, args):
"""Download a remote file.
``args`` could either be a single path, which is interpreted as
path to the remote file to download (to the current working
directory), or a pair ``(remotepath, localdir)``, where localdir
is interpreted as the destination directory.
"""
if isinstance(args, tuple):
remotepath, localdir = args
else:
remotepath = args
localdir = None
try:
self._client.download(remotepath, destdir=localdir, **self._download_kwargs)
cprogress("finished downloading '%s'" % remotepath)
return 0
except KeyboardInterrupt:
cerror("download of '%s' interrupted" % remotepath)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to download '%s': %s: %s" %
(remotepath, type(err).__name__, str(err)))
return 1
def __call__(self, args):
"""Download a remote file.
``args`` could either be a single path, which is interpreted as
path to the remote file to download (to the current working
directory), or a pair ``(remotepath, localdir)``, where localdir
is interpreted as the destination directory.
"""
if isinstance(args, tuple):
remotepath, localdir = args
else:
remotepath = args
localdir = None
try:
self._client.download(remotepath,
destdir=localdir,
**self._download_kwargs)
cprogress("finished downloading '%s'" % remotepath)
return 0
except KeyboardInterrupt:
cerror("download of '%s' interrupted" % remotepath)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to download '%s': %s: %s" %
(remotepath, type(err).__name__, str(err)))
return 1
def main():
"""CLI interface."""
description = "Convert duration in seconds to human readable format."
parser = argparse.ArgumentParser(description=description)
parser.add_argument("-d", "--decimal-digits", metavar="NUM_DIGITS",
nargs="?", type=int, const=2, default=0,
help="""Print digits after the decimal point. By
default the duration is rounded to whole
seconds, but this option enables decimal
digits. If NUM_DIGITS argument is given, print
that many digits after the decimal point; if
this option is specified but no NUM_DIGITS is
given, print 2 digits after the decimal
point.""")
parser.add_argument("-1", "--one-hour-digit", action="store_true",
help="""Only print one hour digit when the
duration is less than ten hours. By default the
hour is zero-padded to two digits.""")
parser.add_argument("seconds", type=float,
help="Total number of seconds. Must be nonnegative.")
args = parser.parse_args()
try:
print(humantime(args.seconds, ndigits=args.decimal_digits,
one_hour_digit=args.one_hour_digit))
except ValueError as err:
cerror(str(err))
return 1
def __call__(self, local_path):
"""Upload a local file."""
try:
self._client.upload(self._directory, local_path, **self._upload_kwargs)
cprogress("finished uploading '%s'" % local_path)
return 0
except KeyboardInterrupt:
cerror("upload of '%s' interrupted" % local_path)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to upload '%s' to '%s': %s: %s" %
(local_path, self._directory, type(err).__name__, str(err)))
return 1
def __call__(self, local_path):
"""Upload a local file."""
try:
self._client.upload(self._directory, local_path,
**self._upload_kwargs)
cprogress("finished uploading '%s'" % local_path)
return 0
except KeyboardInterrupt:
cerror("upload of '%s' interrupted" % local_path)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to upload '%s' to '%s': %s: %s" %
(local_path, self._directory, type(err).__name__, str(err)))
return 1
def _init_client():
"""Init a client or exit with 1.
Print a helpful error message and exit with code 1 if client
initialization somehow fails due to problematic config file.
Returns
-------
onedrive.api.OneDriveAPIClient
"""
try:
return onedrive.api.OneDriveAPIClient()
except OSError as err:
cerror(str(err))
exit(1)
def _init_client():
"""Init a client or exit with 1.
Print a helpful error message and exit with code 1 if client
initialization somehow fails due to problematic config file.
Returns
-------
onedrive.api.OneDriveAPIClient
"""
try:
return onedrive.api.OneDriveAPIClient()
except OSError as err:
cerror(str(err))
exit(1)
def cli_geturl():
"""Get URL CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("path", help="remote path (file or directory)")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
path = args.path
try:
print(client.geturl(path))
return 0
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
return 1
except Exception as err:
cerror("failed to get URL for '%s': %s: %s" %
(path, type(err).__name__, str(err)))
return 1
def cli_metadata():
"""Display metadata CLI."""
parser = argparse.ArgumentParser(description="Dump JSON metadata of item.")
parser.add_argument("path", help="remote path (file or directory)")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
path = args.path
try:
print(json.dumps(client.metadata(path), indent=4))
return 0
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
return 1
except Exception as err:
cerror("failed to get URL for '%s': %s: %s" %
(path, type(err).__name__, str(err)))
return 1
def cli_rmdir():
"""Remove empty directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="DIRECTORY", nargs="+",
help="path of remote directory to remove")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rmdir(path)
cprogress("directory '%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_metadata():
"""Display metadata CLI."""
parser = argparse.ArgumentParser(description="Dump JSON metadata of item.")
parser.add_argument("path", help="remote path (file or directory)")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
path = args.path
try:
print(json.dumps(client.metadata(path), indent=4))
return 0
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
return 1
except Exception as err:
cerror("failed to get URL for '%s': %s: %s" %
(path, type(err).__name__, str(err)))
return 1
def cli_geturl():
"""Get URL CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("path", help="remote path (file or directory)")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
path = args.path
try:
print(client.geturl(path))
return 0
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
return 1
except Exception as err:
cerror("failed to get URL for '%s': %s: %s" %
(path, type(err).__name__, str(err)))
return 1
def run(self):
"""Run the copy operation and monitor status.
The exit code is either 0 or 1, indicating success or failure.
"""
try:
if not self._recursive:
self._client.assert_file(self.src)
self._client.copy(self.src, self.dst,
overwrite=self._overwrite, show_progress=self._show_progress)
cprogress("finished copying '%s' to '%s'" % (self.src, self.dst))
return 0
except KeyboardInterrupt:
cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst))
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to copy '%s' to '%s': %s: %s" %
(self.src, self.dst, type(err).__name__, str(err)))
return 1
def cli_rm():
"""Remove CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="PATH", nargs="+",
help="path of remote item to remove")
parser.add_argument("-r", "-R", "--recursive", action="store_true",
help="remove directories and their contents recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rm(path, recursive=args.recursive)
cprogress("'%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_rmdir():
"""Remove empty directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="DIRECTORY",
nargs="+",
help="path of remote directory to remove")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rmdir(path)
cprogress("directory '%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def run(self):
"""Run the copy operation and monitor status.
The exit code is either 0 or 1, indicating success or failure.
"""
try:
if not self._recursive:
self._client.assert_file(self.src)
self._client.copy(self.src,
self.dst,
overwrite=self._overwrite,
show_progress=self._show_progress)
cprogress("finished copying '%s' to '%s'" % (self.src, self.dst))
return 0
except KeyboardInterrupt:
cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst))
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to copy '%s' to '%s': %s: %s" %
(self.src, self.dst, type(err).__name__, str(err)))
return 1
def cli_mv_or_cp(util, util_name=None):
"""Mimic the behavior of coreutils ``mv`` or ``cp``.
Parameters
----------
util : {"mv", "cp"}
util_name : str, optional
Utility name shown in usage and help text. If omitted, will be
set to the value of ``util``.
"""
usage = """
{util_name} [options] [-T] SOURCE DEST
{util_name} [options] SOURCE... DIRECTORY
{util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name)
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("paths", nargs="+",
help="sources, destination or directory, depending on invocation")
parser.add_argument("-t", "--target-directory", action="store_true",
help="copy all SOURCE arguments into DIRECTORY")
parser.add_argument("-T", "--no-target-directory", action="store_true",
help="treat DEST as exact destination, not directory")
parser.add_argument("-f", "--force", action="store_true",
help="overwrite existing destinations")
if util == "cp":
parser.add_argument("-R", "-r", "--recursive", action="store_true",
help="copy directories recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = None # defer setup
# list of (src, dst) pairs, where dst is the full destination
src_dst_list = []
if args.target_directory and args.no_target_directory:
cfatal_error("conflicting options -t and -T; see %s -h" % util_name)
return 1
if len(args.paths) < 2:
cfatal_error("at least two paths required; see %s -h" % util_name)
return 1
elif len(args.paths) == 2:
# single source item
if args.target_directory:
# mv/cp -t DIRECTORY SOURCE
directory, source = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
elif args.no_target_directory:
# mv/cp -T SOURCE DEST
source, dest = args.paths
else:
# no -t or -T flag
# automatically decide based on whether dest is an existing directory
client = _init_client()
if client.isdir(args.paths[1]):
# mv/cp SOURCE DIRECTORY
source, directory = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
else:
# mv/cp SOURCE DEST
source, dest = args.paths
src_dst_list.append((source, dest))
else:
# multiple source items
if args.no_target_directory:
cerror("option -T cannot be specified when there are multiple source items")
return 1
elif args.target_directory:
# mv/cp -t DIRECTORY SOURCE...
sources = args.paths[1:]
directory = args.paths[0]
else:
# mv/cp SOURCE... DIRECTORY
sources = args.paths[:-1]
directory = args.paths[-1]
src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source)))
for source in sources]
if client is None:
client = _init_client()
# 3, 2, 1, action!
returncode = 0
if util == "mv":
# move each item synchronously
for src_dst_pair in src_dst_list:
src, dst = src_dst_pair
try:
client.move(src, dst, overwrite=args.force)
cprogress("moved '%s' to '%s'" % (src, dst))
except Exception as err:
cerror("failed to move '%s' to '%s': %s: %s" %
(src, dst, type(err).__name__, str(err)))
returncode = 1
else:
# cp is more involved
num_items = len(src_dst_list)
show_progress = (num_items == 1) and zmwangx.pbar.autopbar()
workers = [CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force,
show_progress=show_progress)
for src, dst in src_dst_list]
try:
for worker in workers:
worker.start()
for worker in workers:
worker.join()
if worker.exitcode != 0:
returncode = 1
except KeyboardInterrupt:
returncode = 1
return returncode
def cli_dirupload():
"""Directory upload CLI."""
# TODO: how to handle uploading to an existing and non-empty directory tree?
# TODO: concurrency
# TODO: option to skip creating directories (useful for resuming dirupload)
parser = argparse.ArgumentParser()
parser.add_argument("remotedir",
help="remote *parent* directory to upload to")
parser.add_argument("localdir",
help="path to the local directory to upload")
parser.add_argument("-n",
"--name",
help="""name of the remote directory (by default
it is just the basename of the local
directory)""")
args = parser.parse_args()
localroot = os.path.abspath(args.localdir)
remoteparent = args.remotedir
remotename = args.name if args.name is not None else os.path.basename(
localroot)
remoteroot = posixpath.join(remoteparent, remotename)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localroot):
cfatal_error("'%s' is not an existing local directory" % localroot)
return 1
if not client.isdir(remoteparent):
cfatal_error("'%s' is not an existing remote directory" % remoteparent)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating directories...")
# uploads is a list of tuples (remotedir, localfile, filesize) to upload
# TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions
# TODO: save calls by creating leaves only (topdown false, and use a
# set to keep track of already created relpaths, and once a leaf is
# created, add to the set itself and all its parents)
uploads = []
for localdir, _, files in os.walk(localroot):
normalized_relpath = onedrive.util.normalized_posixpath(
os.path.relpath(localdir, start=localroot))
remotedir = posixpath.normpath(
posixpath.join(remoteroot, normalized_relpath))
client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok?
print(remotedir, file=sys.stderr)
for filename in files:
localfile = os.path.join(localdir, filename)
uploads.append(
(remotedir, localfile, os.path.getsize(localfile)))
# upload files in ascending order of filesize
uploads = sorted(uploads, key=lambda upload: upload[2])
returncode = 0
total = remaining = len(uploads)
total_bytes = remaining_bytes = sum([upload[2] for upload in uploads])
cprogress("uploading %d files..." % total)
for upload in uploads:
remotedir, localfile, filesize = upload
cprogress("remaining: %d/%d files, %s/%s" %
(remaining, total,
zmwangx.humansize.humansize(
remaining_bytes, prefix="iec", unit=""),
zmwangx.humansize.humansize(
total_bytes, prefix="iec", unit="")))
try:
client.upload(remotedir,
localfile,
show_progress=show_progress)
cprogress("finished uploading '%s'" % localfile)
except Exception as err:
cerror("failed to upload '%s' to '%s': %s: %s" %
(localfile, remotedir, type(err).__name__, str(err)))
returncode = 1
remaining -= 1
remaining_bytes -= filesize
return returncode
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_mv_or_cp(util, util_name=None):
"""Mimic the behavior of coreutils ``mv`` or ``cp``.
Parameters
----------
util : {"mv", "cp"}
util_name : str, optional
Utility name shown in usage and help text. If omitted, will be
set to the value of ``util``.
"""
usage = """
{util_name} [options] [-T] SOURCE DEST
{util_name} [options] SOURCE... DIRECTORY
{util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name)
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"paths",
nargs="+",
help="sources, destination or directory, depending on invocation")
parser.add_argument("-t",
"--target-directory",
action="store_true",
help="copy all SOURCE arguments into DIRECTORY")
parser.add_argument("-T",
"--no-target-directory",
action="store_true",
help="treat DEST as exact destination, not directory")
parser.add_argument("-f",
"--force",
action="store_true",
help="overwrite existing destinations")
if util == "cp":
parser.add_argument("-R",
"-r",
"--recursive",
action="store_true",
help="copy directories recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = None # defer setup
# list of (src, dst) pairs, where dst is the full destination
src_dst_list = []
if args.target_directory and args.no_target_directory:
cfatal_error("conflicting options -t and -T; see %s -h" % util_name)
return 1
if len(args.paths) < 2:
cfatal_error("at least two paths required; see %s -h" % util_name)
return 1
elif len(args.paths) == 2:
# single source item
if args.target_directory:
# mv/cp -t DIRECTORY SOURCE
directory, source = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
elif args.no_target_directory:
# mv/cp -T SOURCE DEST
source, dest = args.paths
else:
# no -t or -T flag
# automatically decide based on whether dest is an existing directory
client = _init_client()
if client.isdir(args.paths[1]):
# mv/cp SOURCE DIRECTORY
source, directory = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
else:
# mv/cp SOURCE DEST
source, dest = args.paths
src_dst_list.append((source, dest))
else:
# multiple source items
if args.no_target_directory:
cerror(
"option -T cannot be specified when there are multiple source items"
)
return 1
elif args.target_directory:
# mv/cp -t DIRECTORY SOURCE...
sources = args.paths[1:]
directory = args.paths[0]
else:
# mv/cp SOURCE... DIRECTORY
sources = args.paths[:-1]
directory = args.paths[-1]
src_dst_list = [(source,
posixpath.join(directory, posixpath.basename(source)))
for source in sources]
if client is None:
client = _init_client()
# 3, 2, 1, action!
returncode = 0
if util == "mv":
# move each item synchronously
for src_dst_pair in src_dst_list:
src, dst = src_dst_pair
try:
client.move(src, dst, overwrite=args.force)
cprogress("moved '%s' to '%s'" % (src, dst))
except Exception as err:
cerror("failed to move '%s' to '%s': %s: %s" %
(src, dst, type(err).__name__, str(err)))
returncode = 1
else:
# cp is more involved
num_items = len(src_dst_list)
show_progress = (num_items == 1) and zmwangx.pbar.autopbar()
workers = [
CopyWorker(client,
src,
dst,
recursive=args.recursive,
overwrite=args.force,
show_progress=show_progress)
for src, dst in src_dst_list
]
try:
for worker in workers:
worker.start()
for worker in workers:
worker.join()
if worker.exitcode != 0:
returncode = 1
except KeyboardInterrupt:
returncode = 1
return returncode
def cli_dirdownload():
"""Directory download CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote directory to download")
parser.add_argument("localdir", help="path to the local *parent* directory to download to")
parser.add_argument("-j", "--jobs", type=int, default=8,
help="number of concurrect downloads, use 0 for unlimited; default is 8")
parser.add_argument("--no-check", action="store_true",
help="do not compare checksum of remote and local files")
parser.add_argument("-f", "--fresh", action="store_true",
help="discard any previous failed download")
parser.add_argument("--curl", dest="downloader", action="store_const", const="curl",
help="use curl to download")
parser.add_argument("--wget", dest="downloader", action="store_const", const="wget",
help="use wget to download")
parser.add_argument("-n", "--name",
help="""name of the local directory to create
(by default it is just the basename of the
remote directory)""")
args = parser.parse_args()
remoteroot = args.remotedir
localparent = os.path.abspath(args.localdir)
localname = args.name if args.name is not None else os.path.basename(remoteroot)
localroot = os.path.join(localparent, localname)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localparent):
cfatal_error("'%s' is not an existing local directory" % localparent)
return 1
if not client.isdir(remoteroot):
cfatal_error("'%s' is not an existing remote directory" % remoteroot)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating local directories...")
# downloads is a list of pairs (remotefile, localdir) to download
downloads = []
for remotedir, _, files in client.walk(remoteroot, paths_only=True):
normalized_relpath = onedrive.util.normalized_ospath(
posixpath.relpath(remotedir, start=remoteroot))
localdir = os.path.normpath(os.path.join(localroot, normalized_relpath))
os.makedirs(localdir, exist_ok=True)
print(localdir, file=sys.stderr)
for filename in files:
remotefile = posixpath.join(remotedir, filename)
downloads.append((remotefile, localdir))
num_files = len(downloads)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (num_files == 1) and zmwangx.pbar.autopbar()
download_kwargs = {
"compare_hash": not args.no_check,
"show_progress": show_progress,
"resume": not args.fresh,
"downloader": args.downloader,
}
cprogress("downloading %d files..." % num_files)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
downloader = Downloader(client, download_kwargs=download_kwargs)
returncodes = []
try:
returncodes = pool.map(downloader, downloads, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_dirdownload():
"""Directory download CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote directory to download")
parser.add_argument(
"localdir", help="path to the local *parent* directory to download to")
parser.add_argument(
"-j",
"--jobs",
type=int,
default=8,
help="number of concurrect downloads, use 0 for unlimited; default is 8"
)
parser.add_argument(
"--no-check",
action="store_true",
help="do not compare checksum of remote and local files")
parser.add_argument("-f",
"--fresh",
action="store_true",
help="discard any previous failed download")
parser.add_argument("--curl",
dest="downloader",
action="store_const",
const="curl",
help="use curl to download")
parser.add_argument("--wget",
dest="downloader",
action="store_const",
const="wget",
help="use wget to download")
parser.add_argument("-n",
"--name",
help="""name of the local directory to create
(by default it is just the basename of the
remote directory)""")
args = parser.parse_args()
remoteroot = args.remotedir
localparent = os.path.abspath(args.localdir)
localname = args.name if args.name is not None else os.path.basename(
remoteroot)
localroot = os.path.join(localparent, localname)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localparent):
cfatal_error("'%s' is not an existing local directory" % localparent)
return 1
if not client.isdir(remoteroot):
cfatal_error("'%s' is not an existing remote directory" % remoteroot)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating local directories...")
# downloads is a list of pairs (remotefile, localdir) to download
downloads = []
for remotedir, _, files in client.walk(remoteroot, paths_only=True):
normalized_relpath = onedrive.util.normalized_ospath(
posixpath.relpath(remotedir, start=remoteroot))
localdir = os.path.normpath(
os.path.join(localroot, normalized_relpath))
os.makedirs(localdir, exist_ok=True)
print(localdir, file=sys.stderr)
for filename in files:
remotefile = posixpath.join(remotedir, filename)
downloads.append((remotefile, localdir))
num_files = len(downloads)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (num_files == 1) and zmwangx.pbar.autopbar()
download_kwargs = {
"compare_hash": not args.no_check,
"show_progress": show_progress,
"resume": not args.fresh,
"downloader": args.downloader,
}
cprogress("downloading %d files..." % num_files)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
downloader = Downloader(client, download_kwargs=download_kwargs)
returncodes = []
try:
returncodes = pool.map(downloader, downloads, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_ls():
"""List items CLI."""
description = """\
ls for OneDrive.
By default the long format is used, i.e., for each entry, the following
fields are printed:
type, childcount, size, indentation, name.
* type is a single character, `d' for a directory and `-' for a file;
* childcount is an integer for a directory and `-' for a file;
* size is the total size of the item (the size of a directory is calculated
recursively); this is by default a human-readable string, and can be
switched to plain byte count using the +h, ++human flag;
* indentation only appears in tree mode, where each level of depth is
translated into a four-space indent;
* name is the basename of the item (note this important difference from
POSIX ls, which would show full arguments under some circumstances; this
is considered a defect and might change in the future).
Long format can be turned off using the +l, ++long format, in which case
only indentations and names are printed.
Note that you can turn on **tree mode** using the -t, --tree flag, in which
case full directory trees are printed (on the fly).
There is also the -d flag for toggling directory only mode, similar to -d
of ls(1) or tree(1) (for tree mode).
Please do not rely on the output format of ls to be stable, and especially
do not parse it programatically, since there is no guarantee that it won't
change in the future. Please use the API instead (children, listdir, walk,
walkn, etc.; see the onedrive.api module).
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(description),
prefix_chars="-+")
parser.add_argument("paths", metavar="PATH", nargs="+",
help="remote path(s)")
parser.add_argument("-d", "--directory", action="store_true",
help="""list directories themselves, not their
contents; when used in conjuction with the --tree
option, omit files in a directory tree""")
parser.add_argument("-t", "--tree", action="store_true",
help="display full directory trees")
parser.add_argument("+h", "++human", action="store_false",
help="turn off human readable format")
parser.add_argument("+l", "++long", action="store_false",
help="turn off long format")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
paths = args.paths
dironly = args.directory
tree = args.tree
human = args.human
long = args.long
# wrap the whole thing into a try block, since ls might take a long
# time in tree mode, and the program could be interrupted
returncode = 0
try:
# categorize paths into files and dirs; files will come first
# the files and dirs lists are lists of tuples (path, metadata)
files = []
dirs = []
for path in paths:
try:
metadata = client.metadata(path)
if "file" in metadata:
files.append((path, metadata))
else:
dirs.append((path, metadata))
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
returncode = 1
# first list files, if any
for _, filemetadata in files:
_cli_ls_print_entry(filemetadata, long=long, human=human)
if not dirs:
return returncode
# list directories
# common kwargs
kwargs = {"dironly": dironly, "tree": tree, "human": human, "long": long}
# handle first directory specially due to special blank line annoyance
firstdirpath, firstdirmetadata = dirs[0]
if not dironly or tree:
if files:
print("")
# do not print "dirname:" if we are only listing a single directory
if len(files) != 0 or len(dirs) != 1:
print("%s:" % firstdirpath)
try:
_cli_ls_single_directory(client, firstdirpath, metadata=firstdirmetadata, **kwargs)
except Exception as err:
cerror("failed to list '%s': %s: %s" % (firstdirpath, type(err).__name__, str(err)))
for dirpath, dirmetadata in dirs[1:]:
if not dironly or tree:
print("")
print("%s:" % dirpath)
try:
_cli_ls_single_directory(client, dirpath, metadata=dirmetadata, **kwargs)
except Exception as err:
cerror("failed to list '%s': %s: %s" % (dirpath, type(err).__name__, str(err)))
except KeyboardInterrupt:
cerror("interrupted")
returncode = 1
return returncode
def cli_rename():
"""Batch renaming CLI."""
parser = argparse.ArgumentParser(
description="Batch rename all items in a directory.")
parser.add_argument("-F",
"--files-only",
action="store_true",
help="""only process files (by default direct
subdirectories are also processed)""")
parser.add_argument(
"-d",
"--dry-run",
action="store_true",
help="""print what would be renamed to stderr but don't do them""")
parser.add_argument("-s",
"--show",
action="store_true",
help="""print what was renamed to what""")
parser.add_argument("stmts",
help="""Python statement(s) to be executed on
each filename; the statement(s) should modify
the '_' variable, which holds the filename; note
that the STL 're' module is already imported for
you, and the statement(s) can also make use of a
variable 'n', which is the one-based index of
the filename currently being handled (indexed in
alphabetical order)""")
parser.add_argument("directory", help="path to remote directory")
args = parser.parse_args()
stmts = args.stmts
try:
ast.parse(stmts)
except SyntaxError as err:
cfatal_error("invalid statements '%s': %s" % (stmts, str(err)))
return 1
onedrive.log.logging_setup()
client = _init_client()
directory = args.directory
try:
client.assert_dir(directory)
except Exception as err:
cfatal_error("%s: %s" % (type(err).__name__, str(err)))
return 1
returncode = 0
children = client.children(directory)
index = 0
for child in children:
if args.files_only and "folder" in child:
continue
oldname = child["name"]
index += 1
globaldict = {"re": re}
localdict = {"_": oldname, "n": index}
try:
# pylint: disable=exec-used
exec(stmts, globaldict, localdict)
newname = localdict["_"]
except Exception as err:
cerror("failed to generate new name for '%s': %s: %s" %
(oldname, type(err).__name__, str(err)))
returncode = 1
continue
if oldname == newname:
continue
if args.dry_run or args.show:
print("%s => %s" % (oldname, newname))
if not args.dry_run:
try:
client.move(posixpath.join(directory, oldname),
posixpath.join(directory, newname))
except Exception as err:
cerror("failed to move '%s' for '%s': %s: %s" %
(oldname, newname, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_rename():
"""Batch renaming CLI."""
parser = argparse.ArgumentParser(description="Batch rename all items in a directory.")
parser.add_argument("-F", "--files-only", action="store_true",
help="""only process files (by default direct
subdirectories are also processed)""")
parser.add_argument("-d", "--dry-run", action="store_true",
help="""print what would be renamed to stderr but don't do them""")
parser.add_argument("-s", "--show", action="store_true",
help="""print what was renamed to what""")
parser.add_argument("stmts",
help="""Python statement(s) to be executed on
each filename; the statement(s) should modify
the '_' variable, which holds the filename; note
that the STL 're' module is already imported for
you, and the statement(s) can also make use of a
variable 'n', which is the one-based index of
the filename currently being handled (indexed in
alphabetical order)""")
parser.add_argument("directory", help="path to remote directory")
args = parser.parse_args()
stmts = args.stmts
try:
ast.parse(stmts)
except SyntaxError as err:
cfatal_error("invalid statements '%s': %s" % (stmts, str(err)))
return 1
onedrive.log.logging_setup()
client = _init_client()
directory = args.directory
try:
client.assert_dir(directory)
except Exception as err:
cfatal_error("%s: %s" % (type(err).__name__, str(err)))
return 1
returncode = 0
children = client.children(directory)
index = 0
for child in children:
if args.files_only and "folder" in child:
continue
oldname = child["name"]
index += 1
globaldict = {"re": re}
localdict = {"_": oldname, "n": index}
try:
# pylint: disable=exec-used
exec(stmts, globaldict, localdict)
newname = localdict["_"]
except Exception as err:
cerror("failed to generate new name for '%s': %s: %s" %
(oldname, type(err).__name__, str(err)))
returncode = 1
continue
if oldname == newname:
continue
if args.dry_run or args.show:
print("%s => %s" % (oldname, newname))
if not args.dry_run:
try:
client.move(posixpath.join(directory, oldname),
posixpath.join(directory, newname))
except Exception as err:
cerror("failed to move '%s' for '%s': %s: %s" %
(oldname, newname, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_dirupload():
"""Directory upload CLI."""
# TODO: how to handle uploading to an existing and non-empty directory tree?
# TODO: concurrency
# TODO: option to skip creating directories (useful for resuming dirupload)
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote *parent* directory to upload to")
parser.add_argument("localdir", help="path to the local directory to upload")
parser.add_argument("-n", "--name",
help="""name of the remote directory (by default
it is just the basename of the local
directory)""")
args = parser.parse_args()
localroot = os.path.abspath(args.localdir)
remoteparent = args.remotedir
remotename = args.name if args.name is not None else os.path.basename(localroot)
remoteroot = posixpath.join(remoteparent, remotename)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localroot):
cfatal_error("'%s' is not an existing local directory" % localroot)
return 1
if not client.isdir(remoteparent):
cfatal_error("'%s' is not an existing remote directory" % remoteparent)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating directories...")
# uploads is a list of tuples (remotedir, localfile, filesize) to upload
# TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions
# TODO: save calls by creating leaves only (topdown false, and use a
# set to keep track of already created relpaths, and once a leaf is
# created, add to the set itself and all its parents)
uploads = []
for localdir, _, files in os.walk(localroot):
normalized_relpath = onedrive.util.normalized_posixpath(
os.path.relpath(localdir, start=localroot))
remotedir = posixpath.normpath(posixpath.join(remoteroot, normalized_relpath))
client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok?
print(remotedir, file=sys.stderr)
for filename in files:
localfile = os.path.join(localdir, filename)
uploads.append((remotedir, localfile, os.path.getsize(localfile)))
# upload files in ascending order of filesize
uploads = sorted(uploads, key=lambda upload: upload[2])
returncode = 0
total = remaining = len(uploads)
total_bytes = remaining_bytes = sum([upload[2] for upload in uploads])
cprogress("uploading %d files..." % total)
for upload in uploads:
remotedir, localfile, filesize = upload
cprogress("remaining: %d/%d files, %s/%s" %
(remaining, total,
zmwangx.humansize.humansize(remaining_bytes, prefix="iec", unit=""),
zmwangx.humansize.humansize(total_bytes, prefix="iec", unit="")))
try:
client.upload(remotedir, localfile, show_progress=show_progress)
cprogress("finished uploading '%s'" % localfile)
except Exception as err:
cerror("failed to upload '%s' to '%s': %s: %s" %
(localfile, remotedir, type(err).__name__, str(err)))
returncode = 1
remaining -= 1
remaining_bytes -= filesize
return returncode
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_ls():
"""List items CLI."""
description = """\
ls for OneDrive.
By default the long format is used, i.e., for each entry, the following
fields are printed:
type, childcount, size, indentation, name.
* type is a single character, `d' for a directory and `-' for a file;
* childcount is an integer for a directory and `-' for a file;
* size is the total size of the item (the size of a directory is calculated
recursively); this is by default a human-readable string, and can be
switched to plain byte count using the +h, ++human flag;
* indentation only appears in tree mode, where each level of depth is
translated into a four-space indent;
* name is the basename of the item (note this important difference from
POSIX ls, which would show full arguments under some circumstances; this
is considered a defect and might change in the future).
Long format can be turned off using the +l, ++long format, in which case
only indentations and names are printed.
Note that you can turn on **tree mode** using the -t, --tree flag, in which
case full directory trees are printed (on the fly).
There is also the -d flag for toggling directory only mode, similar to -d
of ls(1) or tree(1) (for tree mode).
Please do not rely on the output format of ls to be stable, and especially
do not parse it programatically, since there is no guarantee that it won't
change in the future. Please use the API instead (children, listdir, walk,
walkn, etc.; see the onedrive.api module).
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(description),
prefix_chars="-+")
parser.add_argument("paths",
metavar="PATH",
nargs="+",
help="remote path(s)")
parser.add_argument("-d",
"--directory",
action="store_true",
help="""list directories themselves, not their
contents; when used in conjuction with the --tree
option, omit files in a directory tree""")
parser.add_argument("-t",
"--tree",
action="store_true",
help="display full directory trees")
parser.add_argument("+h",
"++human",
action="store_false",
help="turn off human readable format")
parser.add_argument("+l",
"++long",
action="store_false",
help="turn off long format")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
paths = args.paths
dironly = args.directory
tree = args.tree
human = args.human
long = args.long
# wrap the whole thing into a try block, since ls might take a long
# time in tree mode, and the program could be interrupted
returncode = 0
try:
# categorize paths into files and dirs; files will come first
# the files and dirs lists are lists of tuples (path, metadata)
files = []
dirs = []
for path in paths:
try:
metadata = client.metadata(path)
if "file" in metadata:
files.append((path, metadata))
else:
dirs.append((path, metadata))
except onedrive.exceptions.FileNotFoundError:
cerror("'%s' not found on OneDrive" % path)
returncode = 1
# first list files, if any
for _, filemetadata in files:
_cli_ls_print_entry(filemetadata, long=long, human=human)
if not dirs:
return returncode
# list directories
# common kwargs
kwargs = {
"dironly": dironly,
"tree": tree,
"human": human,
"long": long
}
# handle first directory specially due to special blank line annoyance
firstdirpath, firstdirmetadata = dirs[0]
if not dironly or tree:
if files:
print("")
# do not print "dirname:" if we are only listing a single directory
if len(files) != 0 or len(dirs) != 1:
print("%s:" % firstdirpath)
try:
_cli_ls_single_directory(client,
firstdirpath,
metadata=firstdirmetadata,
**kwargs)
except Exception as err:
cerror("failed to list '%s': %s: %s" %
(firstdirpath, type(err).__name__, str(err)))
for dirpath, dirmetadata in dirs[1:]:
if not dironly or tree:
print("")
print("%s:" % dirpath)
try:
_cli_ls_single_directory(client,
dirpath,
metadata=dirmetadata,
**kwargs)
except Exception as err:
cerror("failed to list '%s': %s: %s" %
(dirpath, type(err).__name__, str(err)))
except KeyboardInterrupt:
cerror("interrupted")
returncode = 1
return returncode