def setUp(self):
super(CommonFileSystemTest, self).setUp()
self.local_setup()
self.foo_path = URI(self.baseurl) / 'foo'
self.existing_dir = ujoin(self.baseurl, 'foo')
self.existing_file = ujoin(self.baseurl, 'foo', 'foo.txt')
self.non_existing_file = ujoin(self.baseurl, 'bar.txt')
def setUp(self):
super(CommonFileSystemTest, self).setUp()
self.local_setup()
self.foo_path = URI(self.baseurl) / 'foo'
self.existing_dir = ujoin(self.baseurl, 'foo')
self.existing_file = ujoin(self.baseurl, 'foo', 'foo.txt')
self.non_existing_file = ujoin(self.baseurl, 'bar.txt')
def put(self, path_or_tuple, folder_id='me/skydrive', overwrite=True):
"""Upload a file (object), possibly overwriting (default behavior)
a file with the same "name" attribute, if it exists.
First argument can be either path to a local file or tuple
of "(name, file)", where "file" can be either a file-like object
or just a string of bytes.
overwrite option can be set to False to allow two identically-named
files or "ChooseNewName" to let OneDrive derive some similar
unique name. Behavior of this option mimics underlying API."""
if overwrite is not None:
if overwrite is False:
overwrite = 'false'
elif overwrite in ('true', True):
overwrite = None # don't pass it
elif overwrite != 'ChooseNewName':
raise ValueError('overwrite parameter'
' must be True, False or "ChooseNewName".')
name, src = (basename(path_or_tuple), open(path_or_tuple, 'rb')) \
if isinstance(path_or_tuple, types.StringTypes) \
else (path_or_tuple[0], path_or_tuple[1])
return self(ujoin(folder_id, 'files'),
dict(overwrite=overwrite),
method='post',
files=dict(file=(name, src)))
def put(self, path_or_tuple, folder_id='me/skydrive', overwrite=True):
"""Upload a file (object), possibly overwriting (default behavior)
a file with the same "name" attribute, if it exists.
First argument can be either path to a local file or tuple
of "(name, file)", where "file" can be either a file-like object
or just a string of bytes.
overwrite option can be set to False to allow two identically-named
files or "ChooseNewName" to let OneDrive derive some similar
unique name. Behavior of this option mimics underlying API."""
if overwrite is not None:
if overwrite is False:
overwrite = 'false'
elif overwrite in ('true', True):
overwrite = None # don't pass it
elif overwrite != 'ChooseNewName':
raise ValueError('overwrite parameter'
' must be True, False or "ChooseNewName".')
name, src = (basename(path_or_tuple), open(path_or_tuple, 'rb')) \
if isinstance(path_or_tuple, types.StringTypes) \
else (path_or_tuple[0], path_or_tuple[1])
return self(ujoin(folder_id, 'files'), dict(overwrite=overwrite),
method='post', files=dict(file=(name, src)))
def link(self, obj_id, link_type='shared_read_link'):
'''Return a preauthenticated (usable by anyone) link to a
specified object. Object will be considered "shared" by OneDrive,
even if link is never actually used.
link_type can be either "embed" (returns html), "shared_read_link" or "shared_edit_link".'''
assert link_type in ['embed', 'shared_read_link', 'shared_edit_link']
return self(ujoin(obj_id, link_type), method='get')
def get(self, obj_id, byte_range=None):
'''Download and return a file object or a specified byte_range from it.
See HTTP Range header (rfc2616) for possible byte_range formats,
Examples: "0-499" - byte offsets 0-499 (inclusive), "-500" - final 500 bytes.'''
kwz = dict()
if byte_range: kwz['headers'] = dict(Range='bytes={}'.format(byte_range))
return self(ujoin(obj_id, 'content'), dict(download='true'), raw=True, **kwz)
def link(self, obj_id, link_type='shared_read_link'): '''Return a preauthenticated (usable by anyone) link to a specified object. Object will be considered "shared" by OneDrive, even if link is never actually used. link_type can be either "embed" (returns html), "shared_read_link" or "shared_edit_link".''' assert link_type in ['embed', 'shared_read_link', 'shared_edit_link'] return self(ujoin(obj_id, link_type), method='get')
def stor(*subpaths):
MANTA_USER = os.environ['MANTA_USER']
if not subpaths:
return '/%s/stor' % MANTA_USER
subpath = ujoin(*subpaths)
if subpath.startswith("/"):
subpath = subpath[1:]
return "/%s/stor/%s" % (MANTA_USER, subpath)
def stor(*subpaths):
MANTA_USER = os.environ["MANTA_USER"]
if not subpaths:
return "/%s/stor" % MANTA_USER
subpath = ujoin(*subpaths)
if subpath.startswith("/"):
subpath = subpath[1:]
return "/%s/stor/%s" % (MANTA_USER, subpath)
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, 'ls')
self.client.mkdirp(stor(b, "a1"))
self.client.put(stor(b, "a1/a2.txt"), "this is a1/a2.txt")
self.client.mkdirp(stor(b, "a1/b2"))
self.client.put(stor(b, "a1/b2/a3.txt"), "this is a1/b2/a3.txt")
self.client.mkdirp(stor(b, "b1"))
self.client.put(stor(b, "c1.txt"), "this is c1.txt")
def get(self, obj_id, byte_range=None):
"""Download and return a file object or a specified byte_range from it.
See HTTP Range header (rfc2616) for possible byte_range formats,
Examples: "0-499" - byte offsets 0-499 (inclusive),
"-500" - final 500 bytes."""
kwz = dict()
if byte_range:
kwz["headers"] = dict(Range="bytes={}".format(byte_range))
return self(ujoin(obj_id, "content"), dict(download="true"), raw=True, **kwz)
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, "manyfiles")
# If this dir exists already, then save time, don't rebuild it (i.e.
# presuming all the files were left in place).
if self.client.type(stor(b)) != "directory":
self.client.mkdirp(stor(b))
for i in range(1100):
self.client.put(stor(b, "f%05d" % i), "index %d" % i)
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, 'ls')
self.client.mkdirp(stor(b, "a1"))
self.client.put(stor(b, "a1/a2.txt"), "this is a1/a2.txt")
self.client.mkdirp(stor(b, "a1/b2"))
self.client.put(stor(b, "a1/b2/a3.txt"), "this is a1/b2/a3.txt")
self.client.mkdirp(stor(b, "b1"))
self.client.put(stor(b, "c1.txt"), "this is c1.txt")
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, "manyfiles")
# If this dir exists already, then save time, don't rebuild it (i.e.
# presuming all the files were left in place).
if self.client.type(stor(b)) != "directory":
self.client.mkdirp(stor(b))
for i in range(1100):
self.client.put(stor(b, "f%05d" % i), "index %d" % i)
def get(self, obj_id, byte_range=None):
'''Download and return a file object or a specified byte_range from it.
See HTTP Range header (rfc2616) for possible byte_range formats,
Examples: "0-499" - byte offsets 0-499 (inclusive), "-500" - final 500 bytes.'''
kwz = dict()
if byte_range:
kwz['headers'] = dict(Range='bytes={}'.format(byte_range))
return self(ujoin(obj_id, 'content'),
dict(download='true'),
raw=True,
**kwz)
def put( self, path_or_tuple, folder_id='me/skydrive',
overwrite=None, downsize=None, bits_api_fallback=True ):
'''Upload a file (object), possibly overwriting (default behavior)
a file with the same "name" attribute, if it exists.
First argument can be either path to a local file or tuple
of "(name, file)", where "file" can be either a file-like object
or just a string of bytes.
overwrite option can be set to False to allow two identically-named
files or "ChooseNewName" to let OneDrive derive some similar
unique name. Behavior of this option mimics underlying API.
downsize is a true/false API flag, similar to overwrite.
bits_api_fallback can be either True/False or an integer (number of
bytes), and determines whether method will fall back to using BITS API
(as implemented by "put_bits" method) for large files. Default "True"
(bool) value will use non-BITS file size limit (api_put_max_bytes, ~100 MiB)
as a fallback threshold, passing False will force using single-request uploads.'''
api_overwrite = self._translate_api_flag(overwrite, 'overwrite', ['ChooseNewName'])
api_downsize = self._translate_api_flag(downsize, 'downsize')
name, src = self._process_upload_source(path_or_tuple)
if not isinstance(bits_api_fallback, (int, float, long)):
bits_api_fallback = bool(bits_api_fallback)
if bits_api_fallback is not False:
if bits_api_fallback is True: bits_api_fallback = self.api_put_max_bytes
src.seek(0, os.SEEK_END)
if src.tell() > bits_api_fallback:
log.info(
'Falling-back to using BITS API due to file size (%.1f MiB > %.1f MiB)',
*((float(v) / 2**20) for v in [src.tell(), bits_api_fallback]) )
if overwrite is not None and api_overwrite != 'true':
raise NoAPISupportError( 'Passed "overwrite" flag (value: {!r})'
' is not supported by the BITS API (always "true" there)'.format(overwrite) )
if downsize is not None:
log.warn( 'Passed "downsize" flag (value: %r) will not'
' be used with BITS API, as it is not supported there', downsize )
file_id = self.put_bits(path_or_tuple, folder_id=folder_id) # XXX: overwrite/downsize
return self.info(file_id)
return self( ujoin(folder_id, 'files', name),
dict(overwrite=api_overwrite, downsize_photo_uploads=api_downsize),
data=src, method='put', auth_header=True )
def test_clean(self):
client = self.get_client()
try:
client.list_directory(stor(TDIR))
except manta.MantaError as ex:
if ex.code == "ResourceNotFound":
return
else:
raise
# Don't totally wipe, to save time on test re-runs... though
# I'm sure this will surprise at some point.
skips = [stor(TDIR), stor(TDIR, 'manyfiles')]
for mdir, dirs, nondirs in client.walk(stor(TDIR), False):
if mdir in skips:
continue
for nondir in nondirs:
client.delete_object(ujoin(mdir, nondir["name"]))
client.delete_object(mdir)
def walk(self, mtop, topdown=True):
"""`os.walk(path)` for a directory in Manta.
A somewhat limited form in that some of the optional args to
`os.walk` are not supported. Also, instead of dir *names* and file
*names*, the dirents for those are returned. E.g.:
>>> for dirpath, dirents, objents in client.walk('/trent/stor/test'):
... pprint((dirpath, dirents, objents))
('/trent/stor/test',
[{u'mtime': u'2012-12-12T05:40:23Z',
u'name': u'__pycache__',
u'type': u'directory'}],
[{u'etag': u'a5ab3753-c691-4645-9c14-db6653d4f064',
u'mtime': u'2012-12-12T05:40:22Z',
u'name': u'test.py',
u'size': 5627,
u'type': u'object'},
...])
...
@param mtop {Manta dir}
"""
dirents = self.ls(mtop)
mdirs, mnondirs = [], []
for dirent in sorted(dirents.values(), key=itemgetter("name")):
if dirent["type"] == "directory":
mdirs.append(dirent)
else:
mnondirs.append(dirent)
if topdown:
yield mtop, mdirs, mnondirs
for mdir in mdirs:
mpath = ujoin(mtop, mdir["name"])
for x in self.walk(mpath, topdown):
yield x
if not topdown:
yield mtop, mdirs, mnondirs
def walk(self, mtop, topdown=True):
"""`os.walk(path)` for a directory in Manta.
A somewhat limited form in that some of the optional args to
`os.walk` are not supported. Also, instead of dir *names* and file
*names*, the dirents for those are returned. E.g.:
>>> for dirpath, dirents, objents in client.walk('/trent/stor/test'):
... pprint((dirpath, dirents, objents))
('/trent/stor/test',
[{u'mtime': u'2012-12-12T05:40:23Z',
u'name': u'__pycache__',
u'type': u'directory'}],
[{u'etag': u'a5ab3753-c691-4645-9c14-db6653d4f064',
u'mtime': u'2012-12-12T05:40:22Z',
u'name': u'test.py',
u'size': 5627,
u'type': u'object'},
...])
...
@param mtop {Manta dir}
"""
dirents = self.ls(mtop)
mdirs, mnondirs = [], []
for dirent in sorted(dirents.values(), key=itemgetter("name")):
if dirent["type"] == "directory":
mdirs.append(dirent)
else:
mnondirs.append(dirent)
if topdown:
yield mtop, mdirs, mnondirs
for mdir in mdirs:
mpath = ujoin(mtop, mdir["name"])
for x in self.walk(mpath, topdown):
yield x
if not topdown:
yield mtop, mdirs, mnondirs
def put(self, path_or_tuple, folder_id='me/skydrive', overwrite=None, downsize=None):
"""Upload a file (object), possibly overwriting (default behavior)
a file with the same "name" attribute, if it exists.
First argument can be either path to a local file or tuple
of "(name, file)", where "file" can be either a file-like object
or just a string of bytes.
overwrite option can be set to False to allow two identically-named
files or "ChooseNewName" to let OneDrive derive some similar
unique name. Behavior of this option mimics underlying API.
downsize is a true/false API flag, similar to overwrite."""
overwrite = self._translate_api_flag(overwrite, 'overwrite', ['ChooseNewName'])
downsize = self._translate_api_flag(downsize, 'downsize')
name, src = (basename(path_or_tuple), open(path_or_tuple, 'rb')) \
if isinstance(path_or_tuple, types.StringTypes) \
else (path_or_tuple[0], path_or_tuple[1])
return self(ujoin(folder_id, 'files', name),
dict(downsize_photo_uploads=downsize, overwrite=overwrite),
data=src, method='put', auth_header=True)
def main():
import argparse
parser = argparse.ArgumentParser(
description='Tool to manipulate OneDrive contents.')
parser.add_argument('-c', '--config',
metavar='path', default=conf.ConfigMixin.conf_path_default,
help='Writable configuration state-file (yaml).'
' Used to store authorization_code, access and refresh tokens.'
' Should initially contain at least something like "{client: {id: xxx, secret: yyy}}".'
' Default: %(default)s')
parser.add_argument('-p', '--path', action='store_true',
help='Interpret file/folder arguments only as human paths, not ids (default: guess).'
' Avoid using such paths if non-unique "name"'
' attributes of objects in the same parent folder might be used.')
parser.add_argument('-i', '--id', action='store_true',
help='Interpret file/folder arguments only as ids (default: guess).')
parser.add_argument('-k', '--object-key', metavar='spec',
help='If returned data is an object, or a list of objects, only print this key from there.'
' Supplied spec can be a template string for python str.format,'
' assuming that object gets passed as the first argument.'
' Objects that do not have specified key or cannot'
' be formatted using supplied template will be ignored entirely.'
' Example: {0[id]} {0[name]!r} {0[count]:03d} (uploader: {0[from][name]})')
parser.add_argument('-e', '--encoding', metavar='enc', default='utf-8',
help='Use specified encoding (example: utf-8) for CLI input/output.'
' See full list of supported encodings at:'
' http://docs.python.org/2/library/codecs.html#standard-encodings .'
' Pass empty string or "detect" to detect input encoding via'
' chardet module, if available, falling back to utf-8 and terminal encoding for output.'
' Forced utf-8 is used by default, for consistency and due to its ubiquity.')
parser.add_argument('-V', '--version', action='version',
version='python-onedrive {}'.format(onedrive.__version__),
help='Print version number and exit.')
parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')
cmds = parser.add_subparsers(title='Supported operations', dest='call')
cmd = cmds.add_parser('auth', help='Perform user authentication.')
cmd.add_argument('url', nargs='?', help='URL with the authorization_code.')
cmds.add_parser('auth_refresh',
help='Force-refresh OAuth2 access_token.'
' Should never be necessary under normal conditions.')
cmds.add_parser('quota', help='Print quota information.')
cmds.add_parser('user', help='Print user data.')
cmds.add_parser('recent', help='List recently changed objects.')
cmd = cmds.add_parser('info', help='Display object metadata.')
cmd.add_argument('object',
nargs='?', default='me/skydrive',
help='Object to get info on (default: %(default)s).')
cmd = cmds.add_parser('info_set', help='Manipulate object metadata.')
cmd.add_argument('object', help='Object to manipulate metadata for.')
cmd.add_argument('data',
help='JSON mapping of values to set (example: {"name": "new_file_name.jpg"}).')
cmd = cmds.add_parser('link', help='Get a link to a file.')
cmd.add_argument('object', help='Object to get link for.')
cmd.add_argument('-t', '--type', default='shared_read_link',
help='Type of link to request. Possible values'
' (default: %(default)s): shared_read_link, embed, shared_edit_link.')
cmd = cmds.add_parser('ls', help='List folder contents.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to list contents of (default: %(default)s).')
cmd.add_argument('-r', '--range',
metavar='{[offset]-[limit] | limit}',
help='List only specified range of objects inside.'
' Can be either dash-separated "offset-limit" tuple'
' (any of these can be omitted) or a single "limit" number.')
cmd.add_argument('-o', '--objects', action='store_true',
help='Dump full objects, not just name and id.')
cmd = cmds.add_parser('mkdir', help='Create a folder.')
cmd.add_argument('name',
help='Name (or a path consisting of dirname + basename) of a folder to create.')
cmd.add_argument('folder',
nargs='?', default=None,
help='Parent folder (default: me/skydrive).')
cmd.add_argument('-m', '--metadata',
help='JSON mappings of metadata to set for the created folder.'
' Optonal. Example: {"description": "Photos from last trip to Mordor"}')
cmd = cmds.add_parser('get', help='Download file contents.')
cmd.add_argument('file', help='File (object) to read.')
cmd.add_argument('file_dst', nargs='?', help='Name/path to save file (object) as.')
cmd.add_argument('-b', '--byte-range',
help='Specific range of bytes to read from a file (default: read all).'
' Should be specified in rfc2616 Range HTTP header format.'
' Examples: 0-499 (start - 499), -500 (end-500 to end).')
cmd = cmds.add_parser('put', help='Upload a file.')
cmd.add_argument('file', help='Path to a local file to upload.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to put file into (default: %(default)s).')
cmd.add_argument('-n', '--no-overwrite', action='store_true', default=None,
help='Do not overwrite existing files with the same "name" attribute (visible name).'
' Default (and documented) API behavior is to overwrite such files.')
cmd.add_argument('-d', '--no-downsize', action='store_true', default=None,
help='Disable automatic downsizing when uploading a large image.'
' Default (and documented) API behavior is to downsize images.')
cmd.add_argument('-b', '--bits', action='store_true',
help='Force usage of BITS API (uploads via multiple http requests).'
' Default is to only fallback to it for large (wrt API limits) files.')
cmd.add_argument('--bits-frag-bytes',
type=int, metavar='number',
default=api_v5.PersistentOneDriveAPI.api_bits_default_frag_bytes,
help='Fragment size for using BITS API (if used), in bytes. Default: %(default)s')
cmd.add_argument('--bits-do-auth-refresh-before-commit-hack', action='store_true',
help='Do auth_refresh trick before upload session commit request.'
' This is reported to avoid current (as of 2015-01-16) http 5XX errors from the API.'
' See github issue #39, gist with BITS API spec and the README file for more details.')
cmd = cmds.add_parser('cp', help='Copy file to a folder.')
cmd.add_argument('file', help='File (object) to copy.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to copy file to (default: %(default)s).')
cmd = cmds.add_parser('mv', help='Move file to a folder.')
cmd.add_argument('file', help='File (object) to move.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to move file to (default: %(default)s).')
cmd = cmds.add_parser('rm', help='Remove object (file or folder).')
cmd.add_argument('object', nargs='+', help='Object(s) to remove.')
cmd = cmds.add_parser('comments', help='Show comments for a file, object or folder.')
cmd.add_argument('object', help='Object to show comments for.')
cmd = cmds.add_parser('comment_add', help='Add comment for a file, object or folder.')
cmd.add_argument('object', help='Object to add comment for.')
cmd.add_argument('message', help='Comment message to add.')
cmd = cmds.add_parser('comment_delete', help='Delete comment from a file, object or folder.')
cmd.add_argument('comment_id',
help='ID of the comment to remove (use "comments"'
' action to get comment ids along with the messages).')
cmd = cmds.add_parser('tree',
help='Show contents of onedrive (or folder) as a tree of file/folder names.'
' Note that this operation will have to (separately) request a listing of every'
' folder under the specified one, so can be quite slow for large number of these.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to display contents of (default: %(default)s).')
cmd.add_argument('-o', '--objects', action='store_true',
help='Dump full objects, not just name and type.')
optz = parser.parse_args()
if optz.path and optz.id:
parser.error('--path and --id options cannot be used together.')
if optz.encoding.strip('"') in [None, '', 'detect']: optz.encoding = None
if optz.encoding:
global force_encoding
force_encoding = optz.encoding
import codecs
sys.stdin = codecs.getreader(optz.encoding)(sys.stdin)
sys.stdout = codecs.getwriter(optz.encoding)(sys.stdout)
global log
log = logging.getLogger()
logging.basicConfig(level=logging.WARNING
if not optz.debug else logging.DEBUG)
api = api_v5.PersistentOneDriveAPI.from_conf(optz.config)
res = xres = None
resolve_path = ( (lambda s: id_match(s) or api.resolve_path(s))\
if not optz.path else api.resolve_path ) if not optz.id else (lambda obj_id: obj_id)
# Make best-effort to decode all CLI options to unicode
for k, v in vars(optz).viewitems():
if isinstance(v, bytes): setattr(optz, k, decode_obj(v))
elif isinstance(v, list): setattr(optz, k, map(decode_obj, v))
if optz.call == 'auth':
if not optz.url:
print(
'Visit the following URL in any web browser (firefox, chrome, safari, etc),\n'
' authorize there, confirm access permissions, and paste URL of an empty page\n'
' (starting with "https://login.live.com/oauth20_desktop.srf")'
' you will get redirected to in the end.' )
print(
'Alternatively, use the returned (after redirects)'
' URL with "{} auth <URL>" command.\n'.format(sys.argv[0]) )
print('URL to visit: {}\n'.format(api.auth_user_get_url()))
try: import readline # for better compatibility with terminal quirks, see #40
except ImportError: pass
optz.url = raw_input('URL after last redirect: ').strip()
if optz.url:
api.auth_user_process_url(optz.url)
api.auth_get_token()
print('API authorization was completed successfully.')
elif optz.call == 'auth_refresh':
xres = dict(scope_granted=api.auth_get_token())
elif optz.call == 'quota':
df, ds = map(size_units, api.get_quota())
res = dict(free='{:.1f}{}'.format(*df), quota='{:.1f}{}'.format(*ds))
elif optz.call == 'user':
res = api.get_user_data()
elif optz.call == 'recent':
res = api('me/skydrive/recent_docs')['data']
elif optz.call == 'ls':
offset = limit = None
if optz.range:
span = re.search(r'^(\d+)?[-:](\d+)?$', optz.range)
try:
if not span: limit = int(optz.range)
else: offset, limit = map(int, span.groups())
except ValueError:
parser.error(
'--range argument must be in the "[offset]-[limit]"'
' or just "limit" format, with integers as both offset and'
' limit (if not omitted). Provided: {}'.format(optz.range) )
res = sorted(
api.listdir(resolve_path(optz.folder), offset=offset, limit=limit),
key=op.itemgetter('name') )
if not optz.objects: res = map(op.itemgetter('name'), res)
elif optz.call == 'info':
res = api.info(resolve_path(optz.object))
elif optz.call == 'info_set':
xres = api.info_update(resolve_path(optz.object), json.loads(optz.data))
elif optz.call == 'link':
res = api.link(resolve_path(optz.object), optz.type)
elif optz.call == 'comments':
res = api.comments(resolve_path(optz.object))
elif optz.call == 'comment_add':
res = api.comment_add(resolve_path(optz.object), optz.message)
elif optz.call == 'comment_delete':
res = api.comment_delete(optz.comment_id)
elif optz.call == 'mkdir':
name, path = optz.name.replace('\\', '/'), optz.folder
if '/' in name:
name, path_ext = ubasename(name), udirname(name)
path = ujoin(path, path_ext.strip('/')) if path else path_ext
xres = api.mkdir( name=name, folder_id=resolve_path(path),
metadata=optz.metadata and json.loads(optz.metadata) or dict() )
elif optz.call == 'get':
contents = api.get(resolve_path(optz.file), byte_range=optz.byte_range)
if optz.file_dst:
dst_dir = dirname(abspath(optz.file_dst))
if not isdir(dst_dir): os.makedirs(dst_dir)
with open(optz.file_dst, "wb") as dst: dst.write(contents)
else:
sys.stdout.write(contents)
sys.stdout.flush()
elif optz.call == 'put':
dst = optz.folder
if optz.bits_do_auth_refresh_before_commit_hack:
api.api_bits_auth_refresh_before_commit_hack = True
if optz.bits_frag_bytes > 0: api.api_bits_default_frag_bytes = optz.bits_frag_bytes
if dst is not None:
xres = api.put( optz.file, resolve_path(dst),
bits_api_fallback=0 if optz.bits else True, # 0 = "always use BITS"
overwrite=optz.no_overwrite and False, downsize=optz.no_downsize and False )
elif optz.call in ['cp', 'mv']:
argz = map(resolve_path, [optz.file, optz.folder])
xres = (api.move if optz.call == 'mv' else api.copy)(*argz)
elif optz.call == 'rm':
for obj in it.imap(resolve_path, optz.object): xres = api.delete(obj)
elif optz.call == 'tree':
def recurse(obj_id):
node = tree_node()
for obj in api.listdir(obj_id):
# Make sure to dump files as lists with -o,
# not dicts, to make them distinguishable from dirs
res = obj['type'] if not optz.objects else [obj['type'], obj]
node[obj['name']] = recurse(obj['id']) \
if obj['type'] in ['folder', 'album'] else res
return node
root_id = resolve_path(optz.folder)
res = {api.info(root_id)['name']: recurse(root_id)}
else:
parser.error('Unrecognized command: {}'.format(optz.call))
if res is not None: print_result(res, tpl=optz.object_key, file=sys.stdout)
if optz.debug and xres is not None:
buff = io.StringIO()
print_result(xres, file=buff)
log.debug('Call result:\n{0}\n{1}{0}'.format('-' * 20, buff.getvalue()))
def listdir(self, folder_id='me/skydrive', limit=None):
"""Get OneDrive object, representing list of objects in a folder."""
return self(ujoin(folder_id, 'files'), dict(limit=limit))
def comment_add(self, obj_id, message):
"""Add comment message to a specified object."""
return self(ujoin(obj_id, "comments"), method="post", data=dict(message=message), auth_header=True)
def put_bits( self, path_or_tuple,
folder_id=None, folder_path=None, frag_bytes=None,
raw_id=False, chunk_callback=None ):
'''Upload a file (object) using BITS API (via several http requests), possibly
overwriting (default behavior) a file with the same "name" attribute, if it exists.
Unlike "put" method, uploads to "folder_path" (instead of folder_id) are
supported here. Either folder path or id can be specified, but not both.
Passed "chunk_callback" function (if any) will be called after each
uploaded chunk with keyword parameters corresponding to
upload state and BITS session info required to resume it, if necessary.
Returns id of the uploaded file, as retured by the API
if raw_id=True is passed, otherwise in a consistent (with other calls)
"file.{user_id}.{file_id}" format (default).'''
# XXX: overwrite/downsize are not documented/supported here (yet?)
name, src = self._process_upload_source(path_or_tuple)
if folder_id is not None and folder_path is not None:
raise ValueError('Either "folder_id" or "folder_path" can be specified, but not both.')
if folder_id is None and folder_path is None: folder_id = 'me/skydrive'
if folder_id and re.search(r'^me(/.*)$', folder_id): folder_id = self.info(folder_id)['id']
if not frag_bytes: frag_bytes = self.api_bits_default_frag_bytes
user_id = self.get_user_id()
if folder_id: # workaround for API-ids inconsistency between BITS and regular API
match = re.search( r'^(?i)folder.[a-f0-9]+.'
'(?P<user_id>[a-f0-9]+(?P<folder_n>!\d+)?)$', folder_id )
if match and not match.group('folder_n'):
# root folder is a special case and can't seem to be accessed by id
folder_id, folder_path = None, ''
else:
if not match:
raise ValueError('Failed to process folder_id for BITS API: {!r}'.format(folder_id))
folder_id = match.group('user_id')
if folder_id:
url = self.api_bits_url_by_id.format(folder_id=folder_id, user_id=user_id, filename=name)
else:
url = self.api_bits_url_by_path.format(
folder_id=folder_id, user_id=user_id, file_path=ujoin(folder_path, name).lstrip('/') )
code, headers, body = self(
url, method='post', auth_header=True, raw_all=True,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Create-Session',
'BITS-Supported-Protocols': self.api_bits_protocol_id })
h = lambda k,hs=dict((k.lower(), v) for k,v in headers.viewitems()): hs.get(k, '')
checks = [ code == 201,
h('bits-packet-type').lower() == 'ack',
h('bits-protocol').lower() == self.api_bits_protocol_id.lower(),
h('bits-session-id') ]
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Create-Session response', headers, body, checks)
bits_sid = h('bits-session-id')
src.seek(0, os.SEEK_END)
c, src_len = 0, src.tell()
cn = src_len / frag_bytes
if frag_bytes * cn != src_len: cn += 1
src.seek(0)
for n in xrange(1, cn+1):
log.debug( 'Uploading BITS fragment'
' %s / %s (max-size: %.2f MiB)', n, cn, frag_bytes / float(2**20) )
frag = BITSFragment(src, frag_bytes)
c1 = c + frag_bytes
self(
url, method='post', raw=True, data=frag,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Fragment',
'BITS-Session-Id': bits_sid,
'Content-Range': 'bytes {}-{}/{}'.format(c, min(c1, src_len)-1, src_len) })
c = c1
if chunk_callback:
chunk_callback(
bytes_transferred=c, bytes_total=src_len,
chunks_transferred=n, chunks_total=cn,
bits_session_id=bits_sid )
if self.api_bits_auth_refresh_before_commit_hack:
# As per #39 and comments under the gist with the spec,
# apparently this trick fixes occasional http-5XX errors from the API
self.auth_get_token()
code, headers, body = self(
url, method='post', auth_header=True, raw_all=True,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Close-Session',
'BITS-Session-Id': bits_sid })
h = lambda k,hs=dict((k.lower(), v) for k,v in headers.viewitems()): hs.get(k, '')
checks = [code in [200, 201], h('bits-packet-type').lower() == 'ack' ]
# int(h('bits-received-content-range') or 0) == src_len -- documented, but missing
# h('bits-session-id') == bits_sid -- documented, but missing
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Close-Session response', headers, body, checks)
# Workaround for API-ids inconsistency between BITS and regular API
file_id = h('x-resource-id')
if not raw_id: file_id = 'file.{}.{}'.format(user_id, file_id)
return file_id
def put(self,
path_or_tuple,
folder_id='me/skydrive',
overwrite=None,
downsize=None,
bits_api_fallback=True):
'''Upload a file (object), possibly overwriting (default behavior)
a file with the same "name" attribute, if it exists.
First argument can be either path to a local file or tuple
of "(name, file)", where "file" can be either a file-like object
or just a string of bytes.
overwrite option can be set to False to allow two identically-named
files or "ChooseNewName" to let OneDrive derive some similar
unique name. Behavior of this option mimics underlying API.
downsize is a true/false API flag, similar to overwrite.
bits_api_fallback can be either True/False or an integer (number of
bytes), and determines whether method will fall back to using BITS API
(as implemented by "put_bits" method) for large files. Default "True"
(bool) value will use non-BITS file size limit (api_put_max_bytes, ~100 MiB)
as a fallback threshold, passing False will force using single-request uploads.'''
api_overwrite = self._translate_api_flag(overwrite, 'overwrite',
['ChooseNewName'])
api_downsize = self._translate_api_flag(downsize, 'downsize')
name, src = self._process_upload_source(path_or_tuple)
if not isinstance(bits_api_fallback, (int, float, long)):
bits_api_fallback = bool(bits_api_fallback)
if bits_api_fallback is not False:
if bits_api_fallback is True:
bits_api_fallback = self.api_put_max_bytes
src.seek(0, os.SEEK_END)
if src.tell() >= bits_api_fallback:
if bits_api_fallback > 0: # not really a "fallback" in this case
log.info(
'Falling-back to using BITS API due to file size (%.1f MiB > %.1f MiB)',
*((float(v) / 2**20)
for v in [src.tell(), bits_api_fallback]))
if overwrite is not None and api_overwrite != 'true':
raise NoAPISupportError(
'Passed "overwrite" flag (value: {!r})'
' is not supported by the BITS API (always "true" there)'
.format(overwrite))
if downsize is not None:
log.info(
'Passed "downsize" flag (value: %r) will not'
' be used with BITS API, as it is not supported there',
downsize)
file_id = self.put_bits(
path_or_tuple,
folder_id=folder_id) # XXX: overwrite/downsize
return self.info(file_id)
return self(ujoin(folder_id, 'files', name),
dict(overwrite=api_overwrite,
downsize_photo_uploads=api_downsize),
data=src,
method='put',
auth_header=True)
def main():
import argparse
parser = argparse.ArgumentParser(description="Tool to manipulate OneDrive contents.")
parser.add_argument(
"-c",
"--config",
metavar="path",
default=conf.ConfigMixin.conf_path_default,
help="Writable configuration state-file (yaml)."
" Used to store authorization_code, access and refresh tokens."
' Should initially contain at least something like "{client: {id: xxx, secret: yyy}}".'
" Default: %(default)s",
)
parser.add_argument(
"-p",
"--path",
action="store_true",
help="Interpret file/folder arguments only as human paths, not ids (default: guess)."
' Avoid using such paths if non-unique "name"'
" attributes of objects in the same parent folder might be used.",
)
parser.add_argument(
"-i", "--id", action="store_true", help="Interpret file/folder arguments only as ids (default: guess)."
)
parser.add_argument(
"-k",
"--object-key",
metavar="spec",
help="If returned data is an object, or a list of objects, only print this key from there."
" Supplied spec can be a template string for python str.format,"
" assuming that object gets passed as the first argument."
" Objects that do not have specified key or cannot"
" be formatted using supplied template will be ignored entirely."
" Example: {0[id]} {0[name]!r} {0[count]:03d} (uploader: {0[from][name]})",
)
parser.add_argument(
"-e",
"--encoding",
metavar="enc",
default="utf-8",
help="Use specified encoding (example: utf-8) for CLI input/output."
" See full list of supported encodings at:"
" http://docs.python.org/2/library/codecs.html#standard-encodings ."
' Pass empty string or "detect" to detect input encoding via'
" chardet module, if available, falling back to utf-8 and terminal encoding for output."
" Forced utf-8 is used by default, for consistency and due to its ubiquity.",
)
parser.add_argument(
"-V",
"--version",
action="version",
version="python-onedrive {}".format(onedrive.__version__),
help="Print version number and exit.",
)
parser.add_argument("--debug", action="store_true", help="Verbose operation mode.")
cmds = parser.add_subparsers(title="Supported operations", dest="call")
cmd = cmds.add_parser("auth", help="Perform user authentication.")
cmd.add_argument("url", nargs="?", help="URL with the authorization_code.")
cmds.add_parser(
"auth_refresh", help="Force-refresh OAuth2 access_token." " Should never be necessary under normal conditions."
)
cmds.add_parser("quota", help="Print quota information.")
cmds.add_parser("user", help="Print user data.")
cmds.add_parser("recent", help="List recently changed objects.")
cmd = cmds.add_parser("info", help="Display object metadata.")
cmd.add_argument("object", nargs="?", default="me/skydrive", help="Object to get info on (default: %(default)s).")
cmd = cmds.add_parser("info_set", help="Manipulate object metadata.")
cmd.add_argument("object", help="Object to manipulate metadata for.")
cmd.add_argument("data", help='JSON mapping of values to set (example: {"name": "new_file_name.jpg"}).')
cmd = cmds.add_parser("link", help="Get a link to a file.")
cmd.add_argument("object", help="Object to get link for.")
cmd.add_argument(
"-t",
"--type",
default="shared_read_link",
help="Type of link to request. Possible values"
" (default: %(default)s): shared_read_link, embed, shared_edit_link.",
)
cmd = cmds.add_parser("ls", help="List folder contents.")
cmd.add_argument(
"folder", nargs="?", default="me/skydrive", help="Folder to list contents of (default: %(default)s)."
)
cmd.add_argument(
"-r",
"--range",
metavar="{[offset]-[limit] | limit}",
help="List only specified range of objects inside."
' Can be either dash-separated "offset-limit" tuple'
' (any of these can be omitted) or a single "limit" number.',
)
cmd.add_argument("-o", "--objects", action="store_true", help="Dump full objects, not just name and id.")
cmd = cmds.add_parser("mkdir", help="Create a folder.")
cmd.add_argument("name", help="Name (or a path consisting of dirname + basename) of a folder to create.")
cmd.add_argument("folder", nargs="?", default=None, help="Parent folder (default: me/skydrive).")
cmd.add_argument(
"-m",
"--metadata",
help="JSON mappings of metadata to set for the created folder."
' Optonal. Example: {"description": "Photos from last trip to Mordor"}',
)
cmd = cmds.add_parser("get", help="Download file contents.")
cmd.add_argument("file", help="File (object) to read.")
cmd.add_argument("file_dst", nargs="?", help="Name/path to save file (object) as.")
cmd.add_argument(
"-b",
"--byte-range",
help="Specific range of bytes to read from a file (default: read all)."
" Should be specified in rfc2616 Range HTTP header format."
" Examples: 0-499 (start - 499), -500 (end-500 to end).",
)
cmd = cmds.add_parser("put", help="Upload a file.")
cmd.add_argument("file", help="Path to a local file to upload.")
cmd.add_argument("folder", nargs="?", default="me/skydrive", help="Folder to put file into (default: %(default)s).")
cmd.add_argument(
"-n",
"--no-overwrite",
action="store_true",
default=None,
help='Do not overwrite existing files with the same "name" attribute (visible name).'
" Default (and documented) API behavior is to overwrite such files.",
)
cmd.add_argument(
"-d",
"--no-downsize",
action="store_true",
default=None,
help="Disable automatic downsizing when uploading a large image."
" Default (and documented) API behavior is to downsize images.",
)
cmd.add_argument(
"-b",
"--bits",
action="store_true",
help="Force usage of BITS API (uploads via multiple http requests)."
" Default is to only fallback to it for large (wrt API limits) files.",
)
cmd.add_argument(
"--bits-frag-bytes",
type=int,
metavar="number",
default=api_v5.PersistentOneDriveAPI.api_bits_default_frag_bytes,
help="Fragment size for using BITS API (if used), in bytes. Default: %(default)s",
)
cmd.add_argument(
"--bits-do-auth-refresh-before-commit-hack",
action="store_true",
help="Do auth_refresh trick before upload session commit request."
" This is reported to avoid current (as of 2015-01-16) http 5XX errors from the API."
" See github issue #39, gist with BITS API spec and the README file for more details.",
)
cmd = cmds.add_parser("cp", help="Copy file to a folder.")
cmd.add_argument("file", help="File (object) to copy.")
cmd.add_argument("folder", nargs="?", default="me/skydrive", help="Folder to copy file to (default: %(default)s).")
cmd = cmds.add_parser("mv", help="Move file to a folder.")
cmd.add_argument("file", help="File (object) to move.")
cmd.add_argument("folder", nargs="?", default="me/skydrive", help="Folder to move file to (default: %(default)s).")
cmd = cmds.add_parser("rm", help="Remove object (file or folder).")
cmd.add_argument("object", nargs="+", help="Object(s) to remove.")
cmd = cmds.add_parser("comments", help="Show comments for a file, object or folder.")
cmd.add_argument("object", help="Object to show comments for.")
cmd = cmds.add_parser("comment_add", help="Add comment for a file, object or folder.")
cmd.add_argument("object", help="Object to add comment for.")
cmd.add_argument("message", help="Comment message to add.")
cmd = cmds.add_parser("comment_delete", help="Delete comment from a file, object or folder.")
cmd.add_argument(
"comment_id",
help='ID of the comment to remove (use "comments"' " action to get comment ids along with the messages).",
)
cmd = cmds.add_parser(
"tree",
help="Show contents of onedrive (or folder) as a tree of file/folder names."
" Note that this operation will have to (separately) request a listing of every"
" folder under the specified one, so can be quite slow for large number of these.",
)
cmd.add_argument(
"folder", nargs="?", default="me/skydrive", help="Folder to display contents of (default: %(default)s)."
)
cmd.add_argument("-o", "--objects", action="store_true", help="Dump full objects, not just name and type.")
optz = parser.parse_args()
if optz.path and optz.id:
parser.error("--path and --id options cannot be used together.")
if optz.encoding.strip('"') in [None, "", "detect"]:
optz.encoding = None
if optz.encoding:
global force_encoding
force_encoding = optz.encoding
reload(sys)
sys.setdefaultencoding(force_encoding)
global log
log = logging.getLogger()
logging.basicConfig(level=logging.WARNING if not optz.debug else logging.DEBUG)
api = api_v5.PersistentOneDriveAPI.from_conf(optz.config)
res = xres = None
resolve_path = (
((lambda s: id_match(s) or api.resolve_path(s)) if not optz.path else api.resolve_path)
if not optz.id
else (lambda obj_id: obj_id)
)
# Make best-effort to decode all CLI options to unicode
for k, v in vars(optz).viewitems():
if isinstance(v, bytes):
setattr(optz, k, decode_obj(v))
elif isinstance(v, list):
setattr(optz, k, map(decode_obj, v))
if optz.call == "auth":
if not optz.url:
print(
"Visit the following URL in any web browser (firefox, chrome, safari, etc),\n"
" authorize there, confirm access permissions, and paste URL of an empty page\n"
' (starting with "https://login.live.com/oauth20_desktop.srf")'
" you will get redirected to in the end."
)
print(
"Alternatively, use the returned (after redirects)"
' URL with "{} auth <URL>" command.\n'.format(sys.argv[0])
)
print("URL to visit: {}\n".format(api.auth_user_get_url()))
try:
import readline # for better compatibility with terminal quirks, see #40
except ImportError:
pass
optz.url = raw_input("URL after last redirect: ").strip()
if optz.url:
api.auth_user_process_url(optz.url)
api.auth_get_token()
print("API authorization was completed successfully.")
elif optz.call == "auth_refresh":
xres = dict(scope_granted=api.auth_get_token())
elif optz.call == "quota":
df, ds = map(size_units, api.get_quota())
res = dict(free="{:.1f}{}".format(*df), quota="{:.1f}{}".format(*ds))
elif optz.call == "user":
res = api.get_user_data()
elif optz.call == "recent":
res = api("me/skydrive/recent_docs")["data"]
elif optz.call == "ls":
offset = limit = None
if optz.range:
span = re.search(r"^(\d+)?[-:](\d+)?$", optz.range)
try:
if not span:
limit = int(optz.range)
else:
offset, limit = map(int, span.groups())
except ValueError:
parser.error(
'--range argument must be in the "[offset]-[limit]"'
' or just "limit" format, with integers as both offset and'
" limit (if not omitted). Provided: {}".format(optz.range)
)
res = sorted(api.listdir(resolve_path(optz.folder), offset=offset, limit=limit), key=op.itemgetter("name"))
if not optz.objects:
res = map(op.itemgetter("name"), res)
elif optz.call == "info":
res = api.info(resolve_path(optz.object))
elif optz.call == "info_set":
xres = api.info_update(resolve_path(optz.object), json.loads(optz.data))
elif optz.call == "link":
res = api.link(resolve_path(optz.object), optz.type)
elif optz.call == "comments":
res = api.comments(resolve_path(optz.object))
elif optz.call == "comment_add":
res = api.comment_add(resolve_path(optz.object), optz.message)
elif optz.call == "comment_delete":
res = api.comment_delete(optz.comment_id)
elif optz.call == "mkdir":
name, path = optz.name.replace("\\", "/"), optz.folder
if "/" in name:
name, path_ext = ubasename(name), udirname(name)
path = ujoin(path, path_ext.strip("/")) if path else path_ext
xres = api.mkdir(
name=name, folder_id=resolve_path(path), metadata=optz.metadata and json.loads(optz.metadata) or dict()
)
elif optz.call == "get":
contents = api.get(resolve_path(optz.file), byte_range=optz.byte_range)
if optz.file_dst:
dst_dir = dirname(abspath(optz.file_dst))
if not isdir(dst_dir):
os.makedirs(dst_dir)
with open(optz.file_dst, "wb") as dst:
dst.write(contents)
else:
sys.stdout.write(contents)
sys.stdout.flush()
elif optz.call == "put":
dst = optz.folder
if optz.bits_do_auth_refresh_before_commit_hack:
api.api_bits_auth_refresh_before_commit_hack = True
if optz.bits_frag_bytes > 0:
api.api_bits_default_frag_bytes = optz.bits_frag_bytes
if dst is not None:
xres = api.put(
optz.file,
resolve_path(dst),
bits_api_fallback=0 if optz.bits else True, # 0 = "always use BITS"
overwrite=optz.no_overwrite and False,
downsize=optz.no_downsize and False,
)
elif optz.call in ["cp", "mv"]:
argz = map(resolve_path, [optz.file, optz.folder])
xres = (api.move if optz.call == "mv" else api.copy)(*argz)
elif optz.call == "rm":
for obj in it.imap(resolve_path, optz.object):
xres = api.delete(obj)
elif optz.call == "tree":
def recurse(obj_id):
node = tree_node()
for obj in api.listdir(obj_id):
# Make sure to dump files as lists with -o,
# not dicts, to make them distinguishable from dirs
res = obj["type"] if not optz.objects else [obj["type"], obj]
node[obj["name"]] = recurse(obj["id"]) if obj["type"] in ["folder", "album"] else res
return node
root_id = resolve_path(optz.folder)
res = {api.info(root_id)["name"]: recurse(root_id)}
else:
parser.error("Unrecognized command: {}".format(optz.call))
if res is not None:
print_result(res, tpl=optz.object_key, file=sys.stdout)
if optz.debug and xres is not None:
buff = io.StringIO()
print_result(xres, file=buff)
log.debug("Call result:\n{0}\n{1}{0}".format("-" * 20, buff.getvalue()))
def put_bits( self, path_or_tuple,
folder_id=None, folder_path=None, frag_bytes=None, raw_id=False ):
'''Upload a file (object) using BITS API (via several http requests), possibly
overwriting (default behavior) a file with the same "name" attribute, if it exists.
Unlike "put" method, uploads to "folder_path" (instead of folder_id) are
supported here. Either folder path or id can be specified, but not both.
Returns id of the uploaded file, as retured by the API
if raw_id=True is passed, otherwise in a consistent (with other calls)
"file.{user_id}.{file_id}" format (default).'''
# XXX: overwrite/downsize are not documented/supported here yet
name, src = self._process_upload_source(path_or_tuple)
if folder_id is not None and folder_path is not None:
raise ValueError('Either "folder_id" or "folder_path" can be specified, but not both.')
if folder_id is None and folder_path is None: folder_id = 'me/skydrive'
if folder_id and re.search(r'^me(/.*)$', folder_id): folder_id = self.info(folder_id)['id']
if not frag_bytes: frag_bytes = self.api_bits_default_frag_bytes
user_id = self.get_user_id()
get_url = lambda:\
(self.api_bits_url_by_id if folder_id else self.api_bits_url_by_path).format(
folder_id=folder_id, folder_path=folder_path, user_id=user_id, filename=name )
for n in xrange(2):
url = get_url()
try:
code, headers, body = self(
url, method='post', auth_header=True, raw_all=True,
raise_for={404: NoAPISupportError},
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Create-Session',
'BITS-Supported-Protocols': self.api_bits_protocol_id })
except NoAPISupportError as err:
if not folder_id: raise ProtocolError(err.code, *err.args)
else: # XXX: workaround for http-404 on folder-id uploads, should be fixed
log.info('Assuming that BITS API does not support folder_id'
' uploads, falling back to manual folder_id -> folder_path conversion')
folder_path, folder_id_orig = list(), folder_id
for n in xrange(100): # depth limit to prevent inf-loop
info = self.info(folder_id)
folder_id = info['parent_id']
if not folder_id: break
folder_path.append(info['name'])
else:
raise OneDriveInteractionError(
'Path recursion depth exceeded', folder_id_orig, folder_path )
folder_id, folder_path = None, ujoin(*reversed(folder_path))
log.debug('Resolved folder_id %r into path: %r', folder_id_orig, folder_path)
else: break
h = lambda k,hs=dict((k.lower(), v) for k,v in headers.viewitems()): hs.get(k, '')
checks = [ code == 201,
h('bits-packet-type').lower() == 'ack',
h('bits-protocol').lower() == self.api_bits_protocol_id.lower() ]
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Create-Session response', headers, body, checks)
bits_sid = headers['bits-session-id']
src.seek(0, os.SEEK_END)
c, src_len = 0, src.tell()
cn = src_len / frag_bytes
if c * cn != src_len: cn += 1
src.seek(0)
for n in xrange(1, cn+1):
log.debug( 'Uploading BITS fragment'
' %s / %s (max-size: %.2f MiB)', n, cn, frag_bytes / float(2**20) )
frag = BITSFragment(src, frag_bytes)
c1 = c + frag_bytes
self(
url, method='post', raw=True, data=frag,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Fragment',
'BITS-Session-Id': bits_sid,
'Content-Range': 'bytes {}-{}/{}'.format(c, min(c1, src_len)-1, src_len) })
c = c1
code, headers, body = self(
url, method='post', auth_header=True, raw_all=True,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Close-Session',
'BITS-Session-Id': bits_sid })
h = lambda k,hs=dict((k.lower(), v) for k,v in headers.viewitems()): hs.get(k, '')
checks = [code in [200, 201], h('bits-packet-type').lower() == 'ack' ]
# int(h('bits-received-content-range') or 0) == src_len -- documented, but missing
# h('bits-session-id') == bits_sid -- documented, but missing
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Close-Session response', headers, body, checks)
# XXX: workaround for API-ids inconsistency
file_id = h('x-resource-id')
if not raw_id: file_id = 'file.{}.{}'.format(user_id, file_id)
return file_id
def put_bits(self, path_or_tuple, folder_id=None, folder_path=None, frag_bytes=None, raw_id=False):
"""Upload a file (object) using BITS API (via several http requests), possibly
overwriting (default behavior) a file with the same "name" attribute, if it exists.
Unlike "put" method, uploads to "folder_path" (instead of folder_id) are
supported here. Either folder path or id can be specified, but not both.
Returns id of the uploaded file, as retured by the API
if raw_id=True is passed, otherwise in a consistent (with other calls)
"file.{user_id}.{file_id}" format (default)."""
# XXX: overwrite/downsize are not documented/supported here (yet?)
name, src = self._process_upload_source(path_or_tuple)
if folder_id is not None and folder_path is not None:
raise ValueError('Either "folder_id" or "folder_path" can be specified, but not both.')
if folder_id is None and folder_path is None:
folder_id = "me/skydrive"
if folder_id and re.search(r"^me(/.*)$", folder_id):
folder_id = self.info(folder_id)["id"]
if not frag_bytes:
frag_bytes = self.api_bits_default_frag_bytes
user_id = self.get_user_id()
if folder_id: # workaround for API-ids inconsistency between BITS and regular API
match = re.search(r"^(?i)folder.[a-f0-9]+." "(?P<user_id>[a-f0-9]+(?P<folder_n>!\d+)?)$", folder_id)
if match and not match.group("folder_n"):
# root folder is a special case and can't seem to be accessed by id
folder_id, folder_path = None, ""
else:
if not match:
raise ValueError("Failed to process folder_id for BITS API: {!r}".format(folder_id))
folder_id = match.group("user_id")
if folder_id:
url = self.api_bits_url_by_id.format(folder_id=folder_id, user_id=user_id, filename=name)
else:
url = self.api_bits_url_by_path.format(
folder_id=folder_id, user_id=user_id, file_path=ujoin(folder_path, name).lstrip("/")
)
code, headers, body = self(
url,
method="post",
auth_header=True,
raw_all=True,
headers={
"X-Http-Method-Override": "BITS_POST",
"BITS-Packet-Type": "Create-Session",
"BITS-Supported-Protocols": self.api_bits_protocol_id,
},
)
h = lambda k, hs=dict((k.lower(), v) for k, v in headers.viewitems()): hs.get(k, "")
checks = [
code == 201,
h("bits-packet-type").lower() == "ack",
h("bits-protocol").lower() == self.api_bits_protocol_id.lower(),
]
if not all(checks):
raise ProtocolError(code, "Invalid BITS Create-Session response", headers, body, checks)
bits_sid = headers["bits-session-id"]
src.seek(0, os.SEEK_END)
c, src_len = 0, src.tell()
cn = src_len / frag_bytes
if frag_bytes * cn != src_len:
cn += 1
src.seek(0)
for n in xrange(1, cn + 1):
log.debug("Uploading BITS fragment" " %s / %s (max-size: %.2f MiB)", n, cn, frag_bytes / float(2 ** 20))
frag = BITSFragment(src, frag_bytes)
c1 = c + frag_bytes
self(
url,
method="post",
raw=True,
data=frag,
headers={
"X-Http-Method-Override": "BITS_POST",
"BITS-Packet-Type": "Fragment",
"BITS-Session-Id": bits_sid,
"Content-Range": "bytes {}-{}/{}".format(c, min(c1, src_len) - 1, src_len),
},
)
c = c1
if self.api_bits_auth_refresh_before_commit_hack:
# As per #39 and comments under the gist with the spec,
# apparently this trick fixes occasional http-5XX errors from the API
self.auth_get_token()
code, headers, body = self(
url,
method="post",
auth_header=True,
raw_all=True,
headers={
"X-Http-Method-Override": "BITS_POST",
"BITS-Packet-Type": "Close-Session",
"BITS-Session-Id": bits_sid,
},
)
h = lambda k, hs=dict((k.lower(), v) for k, v in headers.viewitems()): hs.get(k, "")
checks = [code in [200, 201], h("bits-packet-type").lower() == "ack"]
# int(h('bits-received-content-range') or 0) == src_len -- documented, but missing
# h('bits-session-id') == bits_sid -- documented, but missing
if not all(checks):
raise ProtocolError(code, "Invalid BITS Close-Session response", headers, body, checks)
# Workaround for API-ids inconsistency between BITS and regular API
file_id = h("x-resource-id")
if not raw_id:
file_id = "file.{}.{}".format(user_id, file_id)
return file_id
def _api_url_join(self, *slugs):
slugs = list(
urllib.quote(
slug.encode('utf-8') if isinstance(slug, unicode) else slug)
for slug in slugs)
return ujoin(*slugs)
def comments(self, obj_id): 'Get OneDrive object representing a list of comments for an object.' return self(ujoin(obj_id, 'comments'))
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, 'find')
self.client.mkdirp(stor(b, "dir1/dir2"))
self.client.put(stor(b, "dir1/obj1.txt"), "this is obj1")
def listdir(self, folder_id='me/skydrive', limit=None):
"""Get OneDrive object, representing list of objects in a folder."""
return self(ujoin(folder_id, 'files'), dict(limit=limit))
def setUp(self):
self.client = self.get_client()
self.base = b = ujoin(TDIR, 'find')
self.client.mkdirp(stor(b, "dir1/dir2"))
self.client.put(stor(b, "dir1/obj1.txt"), "this is obj1")
def listdir(self, folder_id='me/skydrive', limit=None, offset=None): 'Get OneDrive object representing list of objects in a folder.' return self(ujoin(folder_id, 'files'), dict(limit=limit, offset=offset))
def comments(self, obj_id):
"""Get OneDrive object, representing a list of comments
for an object."""
return self(ujoin(obj_id, 'comments'))
def comments(self, obj_id):
"""Get OneDrive object, representing a list of comments
for an object."""
return self(ujoin(obj_id, "comments"))
xres = api.info_update(
resolve_path(optz.object), json.loads(optz.data))
elif optz.call == 'link':
res = api.link(resolve_path(optz.object), optz.type)
elif optz.call == 'comments':
res = api.comments(resolve_path(optz.object))
elif optz.call == 'comment_add':
res = api.comment_add(resolve_path(optz.object), optz.message)
elif optz.call == 'comment_delete':
res = api.comment_delete(optz.comment_id)
elif optz.call == 'mkdir':
name, path = optz.name.replace('\\', '/'), optz.folder
if '/' in name:
name, path_ext = ubasename(name), udirname(name)
path = ujoin(path, path_ext.strip('/')) if path else path_ext
xres = api.mkdir(name=name, folder_id=resolve_path(path),
metadata=optz.metadata and json.loads(optz.metadata) or dict())
elif optz.call == 'get':
contents = api.get(resolve_path(optz.file), byte_range=optz.byte_range)
if optz.file_dst:
dst_dir = dirname(abspath(optz.file_dst))
if not isdir(dst_dir):
os.makedirs(dst_dir)
with open(optz.file_dst, "wb") as dst:
dst.write(contents)
else:
sys.stdout.write(contents)
sys.stdout.flush()
def main():
import argparse
parser = argparse.ArgumentParser(
description='Tool to manipulate OneDrive contents.')
parser.add_argument('-c', '--config',
metavar='path', default=conf.ConfigMixin.conf_path_default,
help='Writable configuration state-file (yaml).'
' Used to store authorization_code, access and refresh tokens.'
' Should initially contain at least something like "{client: {id: xxx, secret: yyy}}".'
' Default: %(default)s')
parser.add_argument('-p', '--path', action='store_true',
help='Interpret file/folder arguments only as human paths, not ids (default: guess).'
' Avoid using such paths if non-unique "name"'
' attributes of objects in the same parent folder might be used.')
parser.add_argument('-i', '--id', action='store_true',
help='Interpret file/folder arguments only as ids (default: guess).')
parser.add_argument('-k', '--object-key', metavar='spec',
help='If returned data is an object, or a list of objects, only print this key from there.'
' Supplied spec can be a template string for python str.format,'
' assuming that object gets passed as the first argument.'
' Objects that do not have specified key or cannot'
' be formatted using supplied template will be ignored entirely.'
' Example: {0[id]} {0[name]!r} {0[count]:03d} (uploader: {0[from][name]})')
parser.add_argument('-e', '--encoding', metavar='enc', default='utf-8',
help='Use specified encoding (example: utf-8) for CLI input/output.'
' See full list of supported encodings at:'
' http://docs.python.org/2/library/codecs.html#standard-encodings .'
' Pass empty string or "detect" to detect input encoding via'
' chardet module, if available, falling back to utf-8 and terminal encoding for output.'
' Forced utf-8 is used by default, for consistency and due to its ubiquity.')
parser.add_argument('-V', '--version', action='version',
version='python-onedrive {}'.format(onedrive.__version__),
help='Print version number and exit.')
parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')
cmds = parser.add_subparsers(title='Supported operations', dest='call')
cmd = cmds.add_parser('auth', help='Perform user authentication.')
cmd.add_argument('url', nargs='?', help='URL with the authorization_code.')
cmds.add_parser('auth_refresh',
help='Force-refresh OAuth2 access_token.'
' Should never be necessary under normal conditions.')
cmds.add_parser('quota', help='Print quota information.')
cmds.add_parser('user', help='Print user data.')
cmds.add_parser('recent', help='List recently changed objects.')
cmd = cmds.add_parser('info', help='Display object metadata.')
cmd.add_argument('object',
nargs='?', default='me/skydrive',
help='Object to get info on (default: %(default)s).')
cmd = cmds.add_parser('info_set', help='Manipulate object metadata.')
cmd.add_argument('object', help='Object to manipulate metadata for.')
cmd.add_argument('data',
help='JSON mapping of values to set (example: {"name": "new_file_name.jpg"}).')
cmd = cmds.add_parser('link', help='Get a link to a file.')
cmd.add_argument('object', help='Object to get link for.')
cmd.add_argument('-t', '--type', default='shared_read_link',
help='Type of link to request. Possible values'
' (default: %(default)s): shared_read_link, embed, shared_edit_link.')
cmd = cmds.add_parser('ls', help='List folder contents.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to list contents of (default: %(default)s).')
cmd.add_argument('-r', '--range',
metavar='{[offset]-[limit] | limit}',
help='List only specified range of objects inside.'
' Can be either dash-separated "offset-limit" tuple'
' (any of these can be omitted) or a single "limit" number.')
cmd.add_argument('-o', '--objects', action='store_true',
help='Dump full objects, not just name and id.')
cmd = cmds.add_parser('mkdir', help='Create a folder.')
cmd.add_argument('name',
help='Name (or a path consisting of dirname + basename) of a folder to create.')
cmd.add_argument('folder',
nargs='?', default=None,
help='Parent folder (default: me/skydrive).')
cmd.add_argument('-m', '--metadata',
help='JSON mappings of metadata to set for the created folder.'
' Optonal. Example: {"description": "Photos from last trip to Mordor"}')
cmd = cmds.add_parser('get', help='Download file contents.')
cmd.add_argument('file', help='File (object) to read.')
cmd.add_argument('file_dst', nargs='?', help='Name/path to save file (object) as.')
cmd.add_argument('-b', '--byte-range',
help='Specific range of bytes to read from a file (default: read all).'
' Should be specified in rfc2616 Range HTTP header format.'
' Examples: 0-499 (start - 499), -500 (end-500 to end).')
cmd = cmds.add_parser('put', help='Upload a file.')
cmd.add_argument('file', help='Path to a local file to upload.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to put file into (default: %(default)s).')
cmd.add_argument('-n', '--no-overwrite', action='store_true', default=None,
help='Do not overwrite existing files with the same "name" attribute (visible name).'
' Default (and documented) API behavior is to overwrite such files.')
cmd.add_argument('-d', '--no-downsize', action='store_true', default=None,
help='Disable automatic downsizing when uploading a large image.'
' Default (and documented) API behavior is to downsize images.')
cmd.add_argument('-b', '--bits', action='store_true',
help='Force usage of BITS API (uploads via multiple http requests).'
' Default is to only fallback to it for large (wrt API limits) files.')
cmd.add_argument('--bits-frag-bytes',
type=int, metavar='number',
default=api_v5.PersistentOneDriveAPI.api_bits_default_frag_bytes,
help='Fragment size for using BITS API (if used), in bytes. Default: %(default)s')
cmd.add_argument('--bits-do-auth-refresh-before-commit-hack', action='store_true',
help='Do auth_refresh trick before upload session commit request.'
' This is reported to avoid current (as of 2015-01-16) http 5XX errors from the API.'
' See github issue #39, gist with BITS API spec and the README file for more details.')
cmd = cmds.add_parser('cp', help='Copy file to a folder.')
cmd.add_argument('file', help='File (object) to copy.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to copy file to (default: %(default)s).')
cmd = cmds.add_parser('mv', help='Move file to a folder.')
cmd.add_argument('file', help='File (object) to move.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to move file to (default: %(default)s).')
cmd = cmds.add_parser('rm', help='Remove object (file or folder).')
cmd.add_argument('object', nargs='+', help='Object(s) to remove.')
cmd = cmds.add_parser('comments', help='Show comments for a file, object or folder.')
cmd.add_argument('object', help='Object to show comments for.')
cmd = cmds.add_parser('comment_add', help='Add comment for a file, object or folder.')
cmd.add_argument('object', help='Object to add comment for.')
cmd.add_argument('message', help='Comment message to add.')
cmd = cmds.add_parser('comment_delete', help='Delete comment from a file, object or folder.')
cmd.add_argument('comment_id',
help='ID of the comment to remove (use "comments"'
' action to get comment ids along with the messages).')
cmd = cmds.add_parser('tree',
help='Show contents of onedrive (or folder) as a tree of file/folder names.'
' Note that this operation will have to (separately) request a listing of every'
' folder under the specified one, so can be quite slow for large number of these.')
cmd.add_argument('folder',
nargs='?', default='me/skydrive',
help='Folder to display contents of (default: %(default)s).')
cmd.add_argument('-o', '--objects', action='store_true',
help='Dump full objects, not just name and type.')
optz = parser.parse_args()
if optz.path and optz.id:
parser.error('--path and --id options cannot be used together.')
if optz.encoding.strip('"') in [None, '', 'detect']: optz.encoding = None
if optz.encoding:
global force_encoding
force_encoding = optz.encoding
import codecs
sys.stdin = codecs.getreader(optz.encoding)(sys.stdin)
sys.stdout = codecs.getwriter(optz.encoding)(sys.stdout)
global log
log = logging.getLogger()
logging.basicConfig(level=logging.WARNING
if not optz.debug else logging.DEBUG)
api = api_v5.PersistentOneDriveAPI.from_conf(optz.config)
res = xres = None
resolve_path = ( (lambda s: id_match(s) or api.resolve_path(s))\
if not optz.path else api.resolve_path ) if not optz.id else (lambda obj_id: obj_id)
# Make best-effort to decode all CLI options to unicode
for k, v in vars(optz).viewitems():
if isinstance(v, bytes): setattr(optz, k, decode_obj(v))
elif isinstance(v, list): setattr(optz, k, map(decode_obj, v))
if optz.call == 'auth':
if not optz.url:
print(
'Visit the following URL in any web browser (firefox, chrome, safari, etc),\n'
' authorize there, confirm access permissions, and paste URL of an empty page\n'
' (starting with "https://login.live.com/oauth20_desktop.srf")'
' you will get redirected to in the end.' )
print(
'Alternatively, use the returned (after redirects)'
' URL with "{} auth <URL>" command.\n'.format(sys.argv[0]) )
print('URL to visit: {}\n'.format(api.auth_user_get_url()))
try: import readline # for better compatibility with terminal quirks, see #40
except ImportError: pass
optz.url = raw_input('URL after last redirect: ').strip()
if optz.url:
api.auth_user_process_url(optz.url)
api.auth_get_token()
print('API authorization was completed successfully.')
elif optz.call == 'auth_refresh':
xres = dict(scope_granted=api.auth_get_token())
elif optz.call == 'quota':
df, ds = map(size_units, api.get_quota())
res = dict(free='{:.1f}{}'.format(*df), quota='{:.1f}{}'.format(*ds))
elif optz.call == 'user':
res = api.get_user_data()
elif optz.call == 'recent':
res = api('me/skydrive/recent_docs')['data']
elif optz.call == 'ls':
offset = limit = None
if optz.range:
span = re.search(r'^(\d+)?[-:](\d+)?$', optz.range)
try:
if not span: limit = int(optz.range)
else: offset, limit = map(int, span.groups())
except ValueError:
parser.error(
'--range argument must be in the "[offset]-[limit]"'
' or just "limit" format, with integers as both offset and'
' limit (if not omitted). Provided: {}'.format(optz.range) )
res = sorted(
api.listdir(resolve_path(optz.folder), offset=offset, limit=limit),
key=op.itemgetter('name') )
if not optz.objects: res = map(op.itemgetter('name'), res)
elif optz.call == 'info':
res = api.info(resolve_path(optz.object))
elif optz.call == 'info_set':
xres = api.info_update(resolve_path(optz.object), json.loads(optz.data))
elif optz.call == 'link':
res = api.link(resolve_path(optz.object), optz.type)
elif optz.call == 'comments':
res = api.comments(resolve_path(optz.object))
elif optz.call == 'comment_add':
res = api.comment_add(resolve_path(optz.object), optz.message)
elif optz.call == 'comment_delete':
res = api.comment_delete(optz.comment_id)
elif optz.call == 'mkdir':
name, path = optz.name.replace('\\', '/'), optz.folder
if '/' in name:
name, path_ext = ubasename(name), udirname(name)
path = ujoin(path, path_ext.strip('/')) if path else path_ext
xres = api.mkdir( name=name, folder_id=resolve_path(path),
metadata=optz.metadata and json.loads(optz.metadata) or dict() )
elif optz.call == 'get':
contents = api.get(resolve_path(optz.file), byte_range=optz.byte_range)
if optz.file_dst:
dst_dir = dirname(abspath(optz.file_dst))
if not isdir(dst_dir): os.makedirs(dst_dir)
with open(optz.file_dst, "wb") as dst: dst.write(contents)
else:
sys.stdout.write(contents)
sys.stdout.flush()
elif optz.call == 'put':
dst = optz.folder
if optz.bits_do_auth_refresh_before_commit_hack:
api.api_bits_auth_refresh_before_commit_hack = True
if optz.bits_frag_bytes > 0: api.api_bits_default_frag_bytes = optz.bits_frag_bytes
if dst is not None:
xres = api.put( optz.file, resolve_path(dst),
bits_api_fallback=0 if optz.bits else True, # 0 = "always use BITS"
overwrite=optz.no_overwrite and False, downsize=optz.no_downsize and False )
elif optz.call in ['cp', 'mv']:
argz = map(resolve_path, [optz.file, optz.folder])
xres = (api.move if optz.call == 'mv' else api.copy)(*argz)
elif optz.call == 'rm':
for obj in it.imap(resolve_path, optz.object): xres = api.delete(obj)
elif optz.call == 'tree':
def recurse(obj_id):
node = tree_node()
for obj in api.listdir(obj_id):
# Make sure to dump files as lists with -o,
# not dicts, to make them distinguishable from dirs
res = obj['type'] if not optz.objects else [obj['type'], obj]
node[obj['name']] = recurse(obj['id']) \
if obj['type'] in ['folder', 'album'] else res
return node
root_id = resolve_path(optz.folder)
res = {api.info(root_id)['name']: recurse(root_id)}
else:
parser.error('Unrecognized command: {}'.format(optz.call))
if res is not None: print_result(res, tpl=optz.object_key, file=sys.stdout)
if optz.debug and xres is not None:
buff = io.StringIO()
print_result(xres, file=buff)
log.debug('Call result:\n{0}\n{1}{0}'.format('-' * 20, buff.getvalue()))
def comment_add(self, obj_id, message):
"""Add comment message to a specified object."""
return self(ujoin(obj_id, 'comments'),
method='post',
data=dict(message=message),
auth_header=True)
def comment_add(self, obj_id, message): 'Add comment message to a specified object.' return self( ujoin(obj_id, 'comments'), method='post', data=dict(message=message), auth_header=True )
def _api_url_join(self, *slugs):
slugs = list(urllib.quote(slug.encode("utf-8") if isinstance(slug, unicode) else slug) for slug in slugs)
return ujoin(*slugs)
def put_bits(self,
path_or_tuple,
folder_id=None,
folder_path=None,
frag_bytes=None,
raw_id=False,
chunk_callback=None):
'''Upload a file (object) using BITS API (via several http requests), possibly
overwriting (default behavior) a file with the same "name" attribute, if it exists.
Unlike "put" method, uploads to "folder_path" (instead of folder_id) are
supported here. Either folder path or id can be specified, but not both.
Passed "chunk_callback" function (if any) will be called after each
uploaded chunk with keyword parameters corresponding to
upload state and BITS session info required to resume it, if necessary.
Returns id of the uploaded file, as retured by the API
if raw_id=True is passed, otherwise in a consistent (with other calls)
"file.{user_id}.{file_id}" format (default).'''
# XXX: overwrite/downsize are not documented/supported here (yet?)
name, src = self._process_upload_source(path_or_tuple)
if folder_id is not None and folder_path is not None:
raise ValueError(
'Either "folder_id" or "folder_path" can be specified, but not both.'
)
if folder_id is None and folder_path is None: folder_id = 'me/skydrive'
if folder_id and re.search(r'^me(/.*)$', folder_id):
folder_id = self.info(folder_id)['id']
if not frag_bytes: frag_bytes = self.api_bits_default_frag_bytes
user_id = self.get_user_id()
if folder_id: # workaround for API-ids inconsistency between BITS and regular API
match = re.search(
r'^(?i)folder.[a-f0-9]+.'
'(?P<user_id>[a-f0-9]+(?P<folder_n>!\d+)?)$', folder_id)
if match and not match.group('folder_n'):
# root folder is a special case and can't seem to be accessed by id
folder_id, folder_path = None, ''
else:
if not match:
raise ValueError(
'Failed to process folder_id for BITS API: {!r}'.
format(folder_id))
folder_id = match.group('user_id')
if folder_id:
url = self.api_bits_url_by_id.format(folder_id=folder_id,
user_id=user_id,
filename=name)
else:
url = self.api_bits_url_by_path.format(folder_id=folder_id,
user_id=user_id,
file_path=ujoin(
folder_path,
name).lstrip('/'))
code, headers, body = self(url,
method='post',
auth_header=True,
raw_all=True,
headers={
'X-Http-Method-Override':
'BITS_POST',
'BITS-Packet-Type':
'Create-Session',
'BITS-Supported-Protocols':
self.api_bits_protocol_id
})
h = lambda k, hs=dict(
(k.lower(), v) for k, v in headers.viewitems()): hs.get(k, '')
checks = [
code == 201,
h('bits-packet-type').lower() == 'ack',
h('bits-protocol').lower() == self.api_bits_protocol_id.lower(),
h('bits-session-id')
]
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Create-Session response',
headers, body, checks)
bits_sid = h('bits-session-id')
src.seek(0, os.SEEK_END)
c, src_len = 0, src.tell()
cn = src_len / frag_bytes
if frag_bytes * cn != src_len: cn += 1
src.seek(0)
for n in xrange(1, cn + 1):
log.debug(
'Uploading BITS fragment'
' %s / %s (max-size: %.2f MiB)', n, cn,
frag_bytes / float(2**20))
frag = BITSFragment(src, frag_bytes)
c1 = c + frag_bytes
self(url,
method='post',
raw=True,
data=frag,
headers={
'X-Http-Method-Override':
'BITS_POST',
'BITS-Packet-Type':
'Fragment',
'BITS-Session-Id':
bits_sid,
'Content-Range':
'bytes {}-{}/{}'.format(c,
min(c1, src_len) - 1, src_len)
})
c = c1
if chunk_callback:
chunk_callback(bytes_transferred=c,
bytes_total=src_len,
chunks_transferred=n,
chunks_total=cn,
bits_session_id=bits_sid)
if self.api_bits_auth_refresh_before_commit_hack:
# As per #39 and comments under the gist with the spec,
# apparently this trick fixes occasional http-5XX errors from the API
self.auth_get_token()
code, headers, body = self(url,
method='post',
auth_header=True,
raw_all=True,
headers={
'X-Http-Method-Override': 'BITS_POST',
'BITS-Packet-Type': 'Close-Session',
'BITS-Session-Id': bits_sid
})
h = lambda k, hs=dict(
(k.lower(), v) for k, v in headers.viewitems()): hs.get(k, '')
checks = [code in [200, 201], h('bits-packet-type').lower() == 'ack']
# int(h('bits-received-content-range') or 0) == src_len -- documented, but missing
# h('bits-session-id') == bits_sid -- documented, but missing
if not all(checks):
raise ProtocolError(code, 'Invalid BITS Close-Session response',
headers, body, checks)
# Workaround for API-ids inconsistency between BITS and regular API
file_id = h('x-resource-id')
if not raw_id: file_id = 'file.{}.{}'.format(user_id, file_id)
return file_id
def listdir(self, folder_id='me/skydrive', limit=None, offset=None):
'Get OneDrive object representing list of objects in a folder.'
return self(ujoin(folder_id, 'files'), dict(limit=limit,
offset=offset))
