def cli_mkdir():
"""Make directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="DIRECTORY", nargs="+",
help="path of remote directory to create")
parser.add_argument("-p", "--parents", action="store_true",
help="no error if existing, make parent directories as needed")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
if args.parents:
metadata = client.makedirs(path, exist_ok=True)
else:
metadata = client.mkdir(path)
cprogress("directory '%s' created at '%s'" %
(path, metadata["webUrl"]))
except Exception as err:
cerror("failed to create directory '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_mkdir():
"""Make directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="DIRECTORY",
nargs="+",
help="path of remote directory to create")
parser.add_argument(
"-p",
"--parents",
action="store_true",
help="no error if existing, make parent directories as needed")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
if args.parents:
metadata = client.makedirs(path, exist_ok=True)
else:
metadata = client.mkdir(path)
cprogress("directory '%s' created at '%s'" %
(path, metadata["webUrl"]))
except Exception as err:
cerror("failed to create directory '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_rm():
"""Remove CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="PATH",
nargs="+",
help="path of remote item to remove")
parser.add_argument(
"-r",
"-R",
"--recursive",
action="store_true",
help="remove directories and their contents recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rm(path, recursive=args.recursive)
cprogress("'%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def __call__(self, args):
"""Download a remote file.
``args`` could either be a single path, which is interpreted as
path to the remote file to download (to the current working
directory), or a pair ``(remotepath, localdir)``, where localdir
is interpreted as the destination directory.
"""
if isinstance(args, tuple):
remotepath, localdir = args
else:
remotepath = args
localdir = None
try:
self._client.download(remotepath,
destdir=localdir,
**self._download_kwargs)
cprogress("finished downloading '%s'" % remotepath)
return 0
except KeyboardInterrupt:
cerror("download of '%s' interrupted" % remotepath)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to download '%s': %s: %s" %
(remotepath, type(err).__name__, str(err)))
return 1
def __call__(self, args):
"""Download a remote file.
``args`` could either be a single path, which is interpreted as
path to the remote file to download (to the current working
directory), or a pair ``(remotepath, localdir)``, where localdir
is interpreted as the destination directory.
"""
if isinstance(args, tuple):
remotepath, localdir = args
else:
remotepath = args
localdir = None
try:
self._client.download(remotepath, destdir=localdir, **self._download_kwargs)
cprogress("finished downloading '%s'" % remotepath)
return 0
except KeyboardInterrupt:
cerror("download of '%s' interrupted" % remotepath)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to download '%s': %s: %s" %
(remotepath, type(err).__name__, str(err)))
return 1
def authorize_client(self):
"""Authorize the client using the code flow."""
# get authorization code
#
# authorization url:
# https://login.live.com/oauth20_authorize.srf?client_id={client_id}&scope={scope}
# &response_type=code&redirect_uri={redirect_uri}
auth_url = urllib.parse.urlunparse((
"https", # scheme
"login.live.com", # netloc
"oauth20_authorize.srf", # path
"", # params
urllib.parse.urlencode({
"client_id": self._client_id,
"scope": "wl.signin wl.offline_access onedrive.readwrite",
"response_type": "code",
"redirect_uri": self._redirect_uri,
}), # query
"", # fragment
))
webbrowser.open(auth_url)
info = ("You are being directed to your default web browser for authorization. "
" When done, copy the URL you are redirected to and paste it back here.")
prompt = "Please enter the redirect URL: "
try:
redirect_url = cprompt(info=info, prompt=prompt)
except EOFError:
raise EOFError("no input for the redirect URL prompt")
code = urllib.parse.parse_qs(urllib.parse.urlparse(redirect_url).query)["code"][0]
# redeem the code
payload = {
"client_id": self._client_id,
"redirect_uri": self._redirect_uri,
"client_secret": self._client_secret,
"code": code,
"grant_type": "authorization_code",
}
headers = {"Content-Type": "application/x-www-form-urlencoded"}
redeem_request = requests.post("https://login.live.com/oauth20_token.srf",
data=payload, headers=headers)
response_json = redeem_request.json()
if redeem_request.status_code == 200 and "access_token" in response_json:
self._access_token = redeem_request.json()["access_token"]
self._refresh_token = redeem_request.json()["refresh_token"]
else:
raise onedrive.exceptions.APIRequestError(response=redeem_request,
request_desc="redeeming request")
# rewrite config file
self._conf["oauth"]["refresh_token"] = self._refresh_token
self._conf.rewrite_configs()
cprogress("Refresh token generated and written.")
def __call__(self, local_path):
"""Upload a local file."""
try:
self._client.upload(self._directory, local_path, **self._upload_kwargs)
cprogress("finished uploading '%s'" % local_path)
return 0
except KeyboardInterrupt:
cerror("upload of '%s' interrupted" % local_path)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to upload '%s' to '%s': %s: %s" %
(local_path, self._directory, type(err).__name__, str(err)))
return 1
def __call__(self, local_path):
"""Upload a local file."""
try:
self._client.upload(self._directory, local_path,
**self._upload_kwargs)
cprogress("finished uploading '%s'" % local_path)
return 0
except KeyboardInterrupt:
cerror("upload of '%s' interrupted" % local_path)
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to upload '%s' to '%s': %s: %s" %
(local_path, self._directory, type(err).__name__, str(err)))
return 1
def cli_rmdir():
"""Remove empty directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="DIRECTORY", nargs="+",
help="path of remote directory to remove")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rmdir(path)
cprogress("directory '%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def run(self):
"""Run the copy operation and monitor status.
The exit code is either 0 or 1, indicating success or failure.
"""
try:
if not self._recursive:
self._client.assert_file(self.src)
self._client.copy(self.src, self.dst,
overwrite=self._overwrite, show_progress=self._show_progress)
cprogress("finished copying '%s' to '%s'" % (self.src, self.dst))
return 0
except KeyboardInterrupt:
cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst))
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to copy '%s' to '%s': %s: %s" %
(self.src, self.dst, type(err).__name__, str(err)))
return 1
def cli_rmdir():
"""Remove empty directory CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths",
metavar="DIRECTORY",
nargs="+",
help="path of remote directory to remove")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rmdir(path)
cprogress("directory '%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def cli_rm():
"""Remove CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("paths", metavar="PATH", nargs="+",
help="path of remote item to remove")
parser.add_argument("-r", "-R", "--recursive", action="store_true",
help="remove directories and their contents recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = _init_client()
returncode = 0
for path in args.paths:
try:
client.rm(path, recursive=args.recursive)
cprogress("'%s' removed from OneDrive" % path)
except Exception as err:
cerror("failed to remove '%s': %s: %s" %
(path, type(err).__name__, str(err)))
returncode = 1
return returncode
def run(self):
"""Run the copy operation and monitor status.
The exit code is either 0 or 1, indicating success or failure.
"""
try:
if not self._recursive:
self._client.assert_file(self.src)
self._client.copy(self.src,
self.dst,
overwrite=self._overwrite,
show_progress=self._show_progress)
cprogress("finished copying '%s' to '%s'" % (self.src, self.dst))
return 0
except KeyboardInterrupt:
cerror("copying '%s' to '%s' interrupted" % (self.src, self.dst))
return 1
except Exception as err:
# catch any exception in a multiprocessing environment
cerror("failed to copy '%s' to '%s': %s: %s" %
(self.src, self.dst, type(err).__name__, str(err)))
return 1
def cli_dirdownload():
"""Directory download CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote directory to download")
parser.add_argument(
"localdir", help="path to the local *parent* directory to download to")
parser.add_argument(
"-j",
"--jobs",
type=int,
default=8,
help="number of concurrect downloads, use 0 for unlimited; default is 8"
)
parser.add_argument(
"--no-check",
action="store_true",
help="do not compare checksum of remote and local files")
parser.add_argument("-f",
"--fresh",
action="store_true",
help="discard any previous failed download")
parser.add_argument("--curl",
dest="downloader",
action="store_const",
const="curl",
help="use curl to download")
parser.add_argument("--wget",
dest="downloader",
action="store_const",
const="wget",
help="use wget to download")
parser.add_argument("-n",
"--name",
help="""name of the local directory to create
(by default it is just the basename of the
remote directory)""")
args = parser.parse_args()
remoteroot = args.remotedir
localparent = os.path.abspath(args.localdir)
localname = args.name if args.name is not None else os.path.basename(
remoteroot)
localroot = os.path.join(localparent, localname)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localparent):
cfatal_error("'%s' is not an existing local directory" % localparent)
return 1
if not client.isdir(remoteroot):
cfatal_error("'%s' is not an existing remote directory" % remoteroot)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating local directories...")
# downloads is a list of pairs (remotefile, localdir) to download
downloads = []
for remotedir, _, files in client.walk(remoteroot, paths_only=True):
normalized_relpath = onedrive.util.normalized_ospath(
posixpath.relpath(remotedir, start=remoteroot))
localdir = os.path.normpath(
os.path.join(localroot, normalized_relpath))
os.makedirs(localdir, exist_ok=True)
print(localdir, file=sys.stderr)
for filename in files:
remotefile = posixpath.join(remotedir, filename)
downloads.append((remotefile, localdir))
num_files = len(downloads)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (num_files == 1) and zmwangx.pbar.autopbar()
download_kwargs = {
"compare_hash": not args.no_check,
"show_progress": show_progress,
"resume": not args.fresh,
"downloader": args.downloader,
}
cprogress("downloading %d files..." % num_files)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
downloader = Downloader(client, download_kwargs=download_kwargs)
returncodes = []
try:
returncodes = pool.map(downloader, downloads, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_upload():
"""Upload CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("directory", help="remote directory to upload to")
parser.add_argument("local_paths",
metavar="PATH",
nargs="+",
help="path(s) of local file(s) to upload")
parser.add_argument("-j",
"--jobs",
type=int,
default=4,
help="""number of concurrect uploads (i.e.,
workers), use 0 for unlimited; default is 4""")
parser.add_argument("-f",
"--force",
action="store_true",
help="overwrite if the remote file already exists")
parser.add_argument("-c",
"--chunk-size",
type=int,
default=10485760,
help="""Size in bytes of each chunk in resumable
upload; default is 10 MiB, and the chunk size
should not exceed 60 MiB""")
parser.add_argument("--base-segment-timeout",
type=float,
default=14,
help="""base timeout for uploading a single
segment (10MiB), with one second added to this
base timeout for each worker; default is 14""")
parser.add_argument("--stream",
action="store_true",
help="""Use streaming workers (that stream each
chunk) instead of regular workers; only use this
if you are running a great number of workers
concurrently, or if you are extremely concerned
about memory usage""")
parser.add_argument("--simple-upload-threshold",
type=int,
default=1048576,
help="""file size threshold (in bytes) for using
chunked, resumable upload API instead of simple,
one shot API (less overhead, good for uploading
a great number of small files); default is 1
MiB, and the threshold should not exceed 100
MiB""")
parser.add_argument("--no-check",
action="store_true",
help="""do not compare checksum of local and
remote files (this prevents you from resuming an
upload in case of a failure)""")
args = parser.parse_args()
num_files = len(args.local_paths)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (jobs == 1) and zmwangx.pbar.autopbar()
upload_kwargs = {
"conflict_behavior": "replace" if args.force else "fail",
"simple_upload_threshold": args.simple_upload_threshold,
"compare_hash": not args.no_check,
"chunk_size": args.chunk_size,
"timeout": args.base_segment_timeout + jobs,
"stream": args.stream,
"show_progress": show_progress,
}
onedrive.log.logging_setup()
client = _init_client()
# check existence of remote directory
directory = args.directory
try:
directory_url = client.geturl(directory)
except onedrive.exceptions.GeneralAPIException as err:
cfatal_error(str(err))
return 1
if show_progress:
cprogress("preparing to upload to '%s'" % directory)
cprogress("directory URL: %s" % directory_url)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
uploader = Uploader(client, directory, upload_kwargs)
returncodes = []
try:
returncodes = pool.map(uploader, args.local_paths, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
def cli_dirdownload():
"""Directory download CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote directory to download")
parser.add_argument("localdir", help="path to the local *parent* directory to download to")
parser.add_argument("-j", "--jobs", type=int, default=8,
help="number of concurrect downloads, use 0 for unlimited; default is 8")
parser.add_argument("--no-check", action="store_true",
help="do not compare checksum of remote and local files")
parser.add_argument("-f", "--fresh", action="store_true",
help="discard any previous failed download")
parser.add_argument("--curl", dest="downloader", action="store_const", const="curl",
help="use curl to download")
parser.add_argument("--wget", dest="downloader", action="store_const", const="wget",
help="use wget to download")
parser.add_argument("-n", "--name",
help="""name of the local directory to create
(by default it is just the basename of the
remote directory)""")
args = parser.parse_args()
remoteroot = args.remotedir
localparent = os.path.abspath(args.localdir)
localname = args.name if args.name is not None else os.path.basename(remoteroot)
localroot = os.path.join(localparent, localname)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localparent):
cfatal_error("'%s' is not an existing local directory" % localparent)
return 1
if not client.isdir(remoteroot):
cfatal_error("'%s' is not an existing remote directory" % remoteroot)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating local directories...")
# downloads is a list of pairs (remotefile, localdir) to download
downloads = []
for remotedir, _, files in client.walk(remoteroot, paths_only=True):
normalized_relpath = onedrive.util.normalized_ospath(
posixpath.relpath(remotedir, start=remoteroot))
localdir = os.path.normpath(os.path.join(localroot, normalized_relpath))
os.makedirs(localdir, exist_ok=True)
print(localdir, file=sys.stderr)
for filename in files:
remotefile = posixpath.join(remotedir, filename)
downloads.append((remotefile, localdir))
num_files = len(downloads)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (num_files == 1) and zmwangx.pbar.autopbar()
download_kwargs = {
"compare_hash": not args.no_check,
"show_progress": show_progress,
"resume": not args.fresh,
"downloader": args.downloader,
}
cprogress("downloading %d files..." % num_files)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
downloader = Downloader(client, download_kwargs=download_kwargs)
returncodes = []
try:
returncodes = pool.map(downloader, downloads, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_mv_or_cp(util, util_name=None):
"""Mimic the behavior of coreutils ``mv`` or ``cp``.
Parameters
----------
util : {"mv", "cp"}
util_name : str, optional
Utility name shown in usage and help text. If omitted, will be
set to the value of ``util``.
"""
usage = """
{util_name} [options] [-T] SOURCE DEST
{util_name} [options] SOURCE... DIRECTORY
{util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name)
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("paths", nargs="+",
help="sources, destination or directory, depending on invocation")
parser.add_argument("-t", "--target-directory", action="store_true",
help="copy all SOURCE arguments into DIRECTORY")
parser.add_argument("-T", "--no-target-directory", action="store_true",
help="treat DEST as exact destination, not directory")
parser.add_argument("-f", "--force", action="store_true",
help="overwrite existing destinations")
if util == "cp":
parser.add_argument("-R", "-r", "--recursive", action="store_true",
help="copy directories recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = None # defer setup
# list of (src, dst) pairs, where dst is the full destination
src_dst_list = []
if args.target_directory and args.no_target_directory:
cfatal_error("conflicting options -t and -T; see %s -h" % util_name)
return 1
if len(args.paths) < 2:
cfatal_error("at least two paths required; see %s -h" % util_name)
return 1
elif len(args.paths) == 2:
# single source item
if args.target_directory:
# mv/cp -t DIRECTORY SOURCE
directory, source = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
elif args.no_target_directory:
# mv/cp -T SOURCE DEST
source, dest = args.paths
else:
# no -t or -T flag
# automatically decide based on whether dest is an existing directory
client = _init_client()
if client.isdir(args.paths[1]):
# mv/cp SOURCE DIRECTORY
source, directory = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
else:
# mv/cp SOURCE DEST
source, dest = args.paths
src_dst_list.append((source, dest))
else:
# multiple source items
if args.no_target_directory:
cerror("option -T cannot be specified when there are multiple source items")
return 1
elif args.target_directory:
# mv/cp -t DIRECTORY SOURCE...
sources = args.paths[1:]
directory = args.paths[0]
else:
# mv/cp SOURCE... DIRECTORY
sources = args.paths[:-1]
directory = args.paths[-1]
src_dst_list = [(source, posixpath.join(directory, posixpath.basename(source)))
for source in sources]
if client is None:
client = _init_client()
# 3, 2, 1, action!
returncode = 0
if util == "mv":
# move each item synchronously
for src_dst_pair in src_dst_list:
src, dst = src_dst_pair
try:
client.move(src, dst, overwrite=args.force)
cprogress("moved '%s' to '%s'" % (src, dst))
except Exception as err:
cerror("failed to move '%s' to '%s': %s: %s" %
(src, dst, type(err).__name__, str(err)))
returncode = 1
else:
# cp is more involved
num_items = len(src_dst_list)
show_progress = (num_items == 1) and zmwangx.pbar.autopbar()
workers = [CopyWorker(client, src, dst, recursive=args.recursive, overwrite=args.force,
show_progress=show_progress)
for src, dst in src_dst_list]
try:
for worker in workers:
worker.start()
for worker in workers:
worker.join()
if worker.exitcode != 0:
returncode = 1
except KeyboardInterrupt:
returncode = 1
return returncode
def cli_dirupload():
"""Directory upload CLI."""
# TODO: how to handle uploading to an existing and non-empty directory tree?
# TODO: concurrency
# TODO: option to skip creating directories (useful for resuming dirupload)
parser = argparse.ArgumentParser()
parser.add_argument("remotedir", help="remote *parent* directory to upload to")
parser.add_argument("localdir", help="path to the local directory to upload")
parser.add_argument("-n", "--name",
help="""name of the remote directory (by default
it is just the basename of the local
directory)""")
args = parser.parse_args()
localroot = os.path.abspath(args.localdir)
remoteparent = args.remotedir
remotename = args.name if args.name is not None else os.path.basename(localroot)
remoteroot = posixpath.join(remoteparent, remotename)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localroot):
cfatal_error("'%s' is not an existing local directory" % localroot)
return 1
if not client.isdir(remoteparent):
cfatal_error("'%s' is not an existing remote directory" % remoteparent)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating directories...")
# uploads is a list of tuples (remotedir, localfile, filesize) to upload
# TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions
# TODO: save calls by creating leaves only (topdown false, and use a
# set to keep track of already created relpaths, and once a leaf is
# created, add to the set itself and all its parents)
uploads = []
for localdir, _, files in os.walk(localroot):
normalized_relpath = onedrive.util.normalized_posixpath(
os.path.relpath(localdir, start=localroot))
remotedir = posixpath.normpath(posixpath.join(remoteroot, normalized_relpath))
client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok?
print(remotedir, file=sys.stderr)
for filename in files:
localfile = os.path.join(localdir, filename)
uploads.append((remotedir, localfile, os.path.getsize(localfile)))
# upload files in ascending order of filesize
uploads = sorted(uploads, key=lambda upload: upload[2])
returncode = 0
total = remaining = len(uploads)
total_bytes = remaining_bytes = sum([upload[2] for upload in uploads])
cprogress("uploading %d files..." % total)
for upload in uploads:
remotedir, localfile, filesize = upload
cprogress("remaining: %d/%d files, %s/%s" %
(remaining, total,
zmwangx.humansize.humansize(remaining_bytes, prefix="iec", unit=""),
zmwangx.humansize.humansize(total_bytes, prefix="iec", unit="")))
try:
client.upload(remotedir, localfile, show_progress=show_progress)
cprogress("finished uploading '%s'" % localfile)
except Exception as err:
cerror("failed to upload '%s' to '%s': %s: %s" %
(localfile, remotedir, type(err).__name__, str(err)))
returncode = 1
remaining -= 1
remaining_bytes -= filesize
return returncode
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_dirupload():
"""Directory upload CLI."""
# TODO: how to handle uploading to an existing and non-empty directory tree?
# TODO: concurrency
# TODO: option to skip creating directories (useful for resuming dirupload)
parser = argparse.ArgumentParser()
parser.add_argument("remotedir",
help="remote *parent* directory to upload to")
parser.add_argument("localdir",
help="path to the local directory to upload")
parser.add_argument("-n",
"--name",
help="""name of the remote directory (by default
it is just the basename of the local
directory)""")
args = parser.parse_args()
localroot = os.path.abspath(args.localdir)
remoteparent = args.remotedir
remotename = args.name if args.name is not None else os.path.basename(
localroot)
remoteroot = posixpath.join(remoteparent, remotename)
onedrive.log.logging_setup()
client = _init_client()
if not os.path.isdir(localroot):
cfatal_error("'%s' is not an existing local directory" % localroot)
return 1
if not client.isdir(remoteparent):
cfatal_error("'%s' is not an existing remote directory" % remoteparent)
return 1
try: # KeyboardInterrupt guard block
show_progress = zmwangx.pbar.autopbar()
cprogress("creating directories...")
# uploads is a list of tuples (remotedir, localfile, filesize) to upload
# TODO: default exclusions (e.g., .DS_Store) and user-specified exclusions
# TODO: save calls by creating leaves only (topdown false, and use a
# set to keep track of already created relpaths, and once a leaf is
# created, add to the set itself and all its parents)
uploads = []
for localdir, _, files in os.walk(localroot):
normalized_relpath = onedrive.util.normalized_posixpath(
os.path.relpath(localdir, start=localroot))
remotedir = posixpath.normpath(
posixpath.join(remoteroot, normalized_relpath))
client.makedirs(remotedir, exist_ok=True) # TODO: exist_ok?
print(remotedir, file=sys.stderr)
for filename in files:
localfile = os.path.join(localdir, filename)
uploads.append(
(remotedir, localfile, os.path.getsize(localfile)))
# upload files in ascending order of filesize
uploads = sorted(uploads, key=lambda upload: upload[2])
returncode = 0
total = remaining = len(uploads)
total_bytes = remaining_bytes = sum([upload[2] for upload in uploads])
cprogress("uploading %d files..." % total)
for upload in uploads:
remotedir, localfile, filesize = upload
cprogress("remaining: %d/%d files, %s/%s" %
(remaining, total,
zmwangx.humansize.humansize(
remaining_bytes, prefix="iec", unit=""),
zmwangx.humansize.humansize(
total_bytes, prefix="iec", unit="")))
try:
client.upload(remotedir,
localfile,
show_progress=show_progress)
cprogress("finished uploading '%s'" % localfile)
except Exception as err:
cerror("failed to upload '%s' to '%s': %s: %s" %
(localfile, remotedir, type(err).__name__, str(err)))
returncode = 1
remaining -= 1
remaining_bytes -= filesize
return returncode
except KeyboardInterrupt:
cerror("interrupted" % localfile)
return 1
def cli_upload():
"""Upload CLI."""
parser = argparse.ArgumentParser()
parser.add_argument("directory", help="remote directory to upload to")
parser.add_argument("local_paths", metavar="PATH", nargs="+",
help="path(s) of local file(s) to upload")
parser.add_argument("-j", "--jobs", type=int, default=4,
help="""number of concurrect uploads (i.e.,
workers), use 0 for unlimited; default is 4""")
parser.add_argument("-f", "--force", action="store_true",
help="overwrite if the remote file already exists")
parser.add_argument("-c", "--chunk-size", type=int, default=10485760,
help="""Size in bytes of each chunk in resumable
upload; default is 10 MiB, and the chunk size
should not exceed 60 MiB""")
parser.add_argument("--base-segment-timeout", type=float, default=14,
help="""base timeout for uploading a single
segment (10MiB), with one second added to this
base timeout for each worker; default is 14""")
parser.add_argument("--stream", action="store_true",
help="""Use streaming workers (that stream each
chunk) instead of regular workers; only use this
if you are running a great number of workers
concurrently, or if you are extremely concerned
about memory usage""")
parser.add_argument("--simple-upload-threshold", type=int, default=1048576,
help="""file size threshold (in bytes) for using
chunked, resumable upload API instead of simple,
one shot API (less overhead, good for uploading
a great number of small files); default is 1
MiB, and the threshold should not exceed 100
MiB""")
parser.add_argument("--no-check", action="store_true",
help="""do not compare checksum of local and
remote files (this prevents you from resuming an
upload in case of a failure)""")
args = parser.parse_args()
num_files = len(args.local_paths)
jobs = min(args.jobs, num_files) if args.jobs > 0 else num_files
show_progress = (jobs == 1) and zmwangx.pbar.autopbar()
upload_kwargs = {
"conflict_behavior": "replace" if args.force else "fail",
"simple_upload_threshold": args.simple_upload_threshold,
"compare_hash": not args.no_check,
"chunk_size": args.chunk_size,
"timeout": args.base_segment_timeout + jobs,
"stream": args.stream,
"show_progress": show_progress,
}
onedrive.log.logging_setup()
client = _init_client()
# check existence of remote directory
directory = args.directory
try:
directory_url = client.geturl(directory)
except onedrive.exceptions.GeneralAPIException as err:
cfatal_error(str(err))
return 1
if show_progress:
cprogress("preparing to upload to '%s'" % directory)
cprogress("directory URL: %s" % directory_url)
with multiprocessing.Pool(processes=jobs, maxtasksperchild=1) as pool:
uploader = Uploader(client, directory, upload_kwargs)
returncodes = []
try:
returncodes = pool.map(uploader, args.local_paths, chunksize=1)
except KeyboardInterrupt:
returncodes.append(1)
return 1 if 1 in returncodes else 0
def cli_mv_or_cp(util, util_name=None):
"""Mimic the behavior of coreutils ``mv`` or ``cp``.
Parameters
----------
util : {"mv", "cp"}
util_name : str, optional
Utility name shown in usage and help text. If omitted, will be
set to the value of ``util``.
"""
usage = """
{util_name} [options] [-T] SOURCE DEST
{util_name} [options] SOURCE... DIRECTORY
{util_name} [options] -t DIRECTORY SOURCE...""".format(util_name=util_name)
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"paths",
nargs="+",
help="sources, destination or directory, depending on invocation")
parser.add_argument("-t",
"--target-directory",
action="store_true",
help="copy all SOURCE arguments into DIRECTORY")
parser.add_argument("-T",
"--no-target-directory",
action="store_true",
help="treat DEST as exact destination, not directory")
parser.add_argument("-f",
"--force",
action="store_true",
help="overwrite existing destinations")
if util == "cp":
parser.add_argument("-R",
"-r",
"--recursive",
action="store_true",
help="copy directories recursively")
args = parser.parse_args()
onedrive.log.logging_setup()
client = None # defer setup
# list of (src, dst) pairs, where dst is the full destination
src_dst_list = []
if args.target_directory and args.no_target_directory:
cfatal_error("conflicting options -t and -T; see %s -h" % util_name)
return 1
if len(args.paths) < 2:
cfatal_error("at least two paths required; see %s -h" % util_name)
return 1
elif len(args.paths) == 2:
# single source item
if args.target_directory:
# mv/cp -t DIRECTORY SOURCE
directory, source = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
elif args.no_target_directory:
# mv/cp -T SOURCE DEST
source, dest = args.paths
else:
# no -t or -T flag
# automatically decide based on whether dest is an existing directory
client = _init_client()
if client.isdir(args.paths[1]):
# mv/cp SOURCE DIRECTORY
source, directory = args.paths
dest = posixpath.join(directory, posixpath.basename(source))
else:
# mv/cp SOURCE DEST
source, dest = args.paths
src_dst_list.append((source, dest))
else:
# multiple source items
if args.no_target_directory:
cerror(
"option -T cannot be specified when there are multiple source items"
)
return 1
elif args.target_directory:
# mv/cp -t DIRECTORY SOURCE...
sources = args.paths[1:]
directory = args.paths[0]
else:
# mv/cp SOURCE... DIRECTORY
sources = args.paths[:-1]
directory = args.paths[-1]
src_dst_list = [(source,
posixpath.join(directory, posixpath.basename(source)))
for source in sources]
if client is None:
client = _init_client()
# 3, 2, 1, action!
returncode = 0
if util == "mv":
# move each item synchronously
for src_dst_pair in src_dst_list:
src, dst = src_dst_pair
try:
client.move(src, dst, overwrite=args.force)
cprogress("moved '%s' to '%s'" % (src, dst))
except Exception as err:
cerror("failed to move '%s' to '%s': %s: %s" %
(src, dst, type(err).__name__, str(err)))
returncode = 1
else:
# cp is more involved
num_items = len(src_dst_list)
show_progress = (num_items == 1) and zmwangx.pbar.autopbar()
workers = [
CopyWorker(client,
src,
dst,
recursive=args.recursive,
overwrite=args.force,
show_progress=show_progress)
for src, dst in src_dst_list
]
try:
for worker in workers:
worker.start()
for worker in workers:
worker.join()
if worker.exitcode != 0:
returncode = 1
except KeyboardInterrupt:
returncode = 1
return returncode