def delete(self, path, original_version_id=None):
"""Deletes a file or directory from onedrive.
Ondrive first checks if the version_id, which makes sense, but our tests require us to
throw a 404. In order to achieve this, the metadata of the item is fetched first.
"""
# This will make a call to the api, which will throw 404 if the item is
# no longer there.
item_meta = self.get_item_meta(path)
if original_version_id is not None and original_version_id != 'is_dir':
# since the If-Match header are not working reliable we use the
# data from the response for checking if there is a conflict
try:
version_id = self.version_id_from_meta(item_meta)
except KeyError:
# Raise if the meta does not contain the required keys to return a vid.
# TODO: is it better to just return None in version_id_from_meta?
raise jars.VersionIdNotMatchingError(
version_a=original_version_id, version_b=None)
if version_id != original_version_id:
raise jars.VersionIdNotMatchingError(
version_a=original_version_id, version_b=version_id)
drive_id, item_id = self.api.extract_ids(item_meta)
endpoint = item_endpoint(drive_id=drive_id, item_id=item_id)
self.api.delete(endpoint=endpoint, status_check=raise_for_onedrive)
def move(self,
source,
target,
expected_source_vid=None,
expected_target_vid=None):
"""https://dev.onedrive.com/items/move.htm
"""
# First handle the source
source_meta = self.get_item_meta(source)
if expected_source_vid is not None and expected_source_vid != 'is_dir':
# check it here
if self.version_id_from_meta(source_meta) != expected_source_vid:
raise jars.VersionIdNotMatchingError(self.storage_id,
expected_source_vid)
# Then handle the target
target_dir_meta = self.get_item_meta(target[:-1], safe=True)
target_drive_id, target_item_id = self.api.extract_ids(target_dir_meta)
target_children = self.api.get_children(drive_id=target_drive_id,
item_id=target_item_id)
for child in target_children['value']:
if child['name'] == target[-1]:
target_meta = child
break
if expected_target_vid is not None and expected_target_vid != 'is_dir':
if self.version_id_from_meta(target_meta) != expected_target_vid:
raise jars.VersionIdNotMatchingError(self.storage_id,
expected_target_vid)
path_params = {
'parentReference': {
'id': target_item_id,
'driveId': target_drive_id,
},
'name': target[-1],
'@name.conflictBehavior': 'replace'
}
source_drive_id, source_item_id = self.api.extract_ids(source_meta)
source_endpoint = item_endpoint(drive_id=source_drive_id,
item_id=source_item_id)
item_meta = self.api.patch(endpoint=source_endpoint,
json=path_params,
status_check=raise_for_onedrive).json()
if is_dir(item_meta):
return jars.IS_DIR
else:
return self.version_id_from_meta(item_meta)
def delete(self, path, original_version_id):
""" Delete can be done in 2 ways on GoogleDrive: either with a permanent delete by using
https://developers.google.com/drive/v3/reference/files/delete or set trashed to true with a
patch https://developers.google.com/drive/v3/reference/files/update.
"""
file_id = self._path_to_file_id(path)
file_url = urllib.parse.urljoin(self.base_url, 'files/' + str(file_id))
# fetch current version id since Google Drive has node If-Match header
if original_version_id is not None:
version_id_resp = self.oauth_session.get(
file_url, params={'fields': 'mimeType,' + VERSION_FIELD})
version_id_resp.raise_for_status()
response = version_id_resp.json()
if response['mimeType'] == 'application/vnd.google-apps.folder':
version_id = 'is_dir'
else:
version_id = response[VERSION_FIELD]
if version_id != original_version_id:
raise jars.VersionIdNotMatchingError(
self.storage_id,
version_a=version_id,
version_b=original_version_id)
resp = self.oauth_session.patch(file_url, json={'trashed': True})
resp.raise_for_status()
def raise_for_onedrive(response):
"""Raise a VersionIdNotMatchingErrorConvert if server returns `if-match` or `if=none-match`."""
# if the format of the version id is not correct the server throws a 400
if response.status_code == 400:
if 'if-match' in response.text.lower() or \
'if-none-match' in response.text.lower():
raise jars.VersionIdNotMatchingError('')
raise_for_status(response)
def _verify_version_id(self, path, version_id):
current_meta = self._get_meta(path)
if version_id == jars.FOLDER_VERSION_ID:
return current_meta
current_version_id = self._version_id_from_meta(current_meta)
if current_version_id != version_id:
raise jars.VersionIdNotMatchingError(storage_id=self.storage_id,
version_a=version_id,
version_b=current_version_id)
return current_meta
def _check_version_id(self, file_id, expected_version_id, path=None):
url = urllib.parse.urljoin(self.base_url, 'files/{}'.format(file_id))
if expected_version_id is not None:
response = self.oauth_session.get(
url, params={'fields': 'mimeType,' + VERSION_FIELD})
if expected_version_id == 'is_dir' \
and response.json()['mimeType'] == MIMETYPE_FOLDER:
# its fine, dir to dir
return
response.raise_for_status()
if response.json()[VERSION_FIELD] != expected_version_id:
raise jars.VersionIdNotMatchingError(
self.storage_id,
path=path,
version_b=expected_version_id,
version_a=response.json()[VERSION_FIELD])
def write(self, path, file_obj, original_version_id=None, size=None):
"""https://dev.onedrive.com/items/upload.htm
"""
if size is None:
raise ValueError('OneDrive requires a size to write')
# seperate path into filename and parent_path
filename = path[-1]
parent_path = path[:-1]
# Extract the drive_id and parent_id required to create an item
if original_version_id:
path_meta_list = list(self.api.iter_path(path))
metadata = path_meta_list[-1]
if self.version_id_from_meta(metadata) != original_version_id:
raise jars.VersionIdNotMatchingError(self.storage_id,
original_version_id)
parent_meta = path_meta_list[-2]
drive_id, parent_id = self.api.extract_ids(parent_meta)
else:
# Is the relevant information in the model ?
try:
parent_node = self.model.get_node(parent_path)
drive_id = parent_node.props['_drive_id']
parent_id = parent_node.props['_id']
except KeyError:
# The information is not in the model, try the api.
parent_meta_list = list(self.api.iter_dir_safe(parent_path))
# The last meta in the list is the parent
parent_meta = parent_meta_list[-1]
drive_id, parent_id = self.api.extract_ids(parent_meta)
response = self.api.upload(drive_id=drive_id,
parent_id=parent_id,
filename=filename,
file_obj=file_obj,
size=size)
return self.version_id_from_meta(response.json())
def open_read(self, path, expected_version_id=None):
"""Downloads the file as seen from a filesystems perspective.
Try to extract the information required information from the tree, otherwise fall back to
the server.
https://dev.onedrive.com/items/download.htm
"""
metadata = self.get_item_meta(path)
# Check that the version id is correct
if expected_version_id is not None:
# headers['If-Match'] = expected_version_id
if self.version_id_from_meta(metadata) != expected_version_id:
raise jars.VersionIdNotMatchingError(self.storage_id,
expected_version_id)
# download using the information provided in the metadata.
drive_id, item_id = self.api.extract_ids(metadata)
return self.api.download(drive_id=drive_id, item_id=item_id)
def open_read(self, path, expected_version_id=None):
"""Download the file to the filesystem.
:param path: a path represented by a list of strings
:param expected_version_id
:returns: file like object representing of the given file on storage.
See :py:class:~`BasicStorage` for full documentation.
"""
url = self.urls.download(path)
headers = {'If-Match': expected_version_id}
try:
response = self.api.download(url, headers=headers)
except HTTPError as error:
if error.response.status_code == 404:
raise FileNotFoundError
elif error.response.status_code == 412:
raise jars.VersionIdNotMatchingError()
else:
raise error
return response.raw
def write(self, path, file_obj, original_version_id=None, size=0):
"""
https://developers.google.com/drive/v3/web/manage-uploads
"""
# pylint: disable=too-many-locals,too-many-statements
parent_id = None
node_id = None
http_verb = ''
with self.model.lock:
with contextlib.suppress(KeyError):
parent = self.model.get_node(path[:-1])
parent_id = parent.props['_id']
with contextlib.suppress(KeyError):
node = parent.get_node([path[-1]])
node_id = node.props['_id']
# create parent and fetch id
if parent_id is None:
self.make_dir(path[:-1])
with self.model.lock:
parent = self.model.get_node(path[:-1])
parent_id = parent.props['_id']
url = 'https://www.googleapis.com/upload/drive/v3/files'
# check version id, if there is one to check
if original_version_id is not None:
res = self.oauth_session.get(
'https://www.googleapis.com/drive/v3/files/{}'.format(node_id),
params={'fields': VERSION_FIELD})
res.raise_for_status()
if res.json()[VERSION_FIELD] != original_version_id:
raise jars.VersionIdNotMatchingError(
self.storage_id,
version_a=res.json()[VERSION_FIELD],
version_b=original_version_id)
body = {} # 'parents': [parent_id], 'name': path[-1]}
if node_id is not None:
http_verb = 'PATCH'
body['fileId'] = node_id
url += '/' + node_id
else:
http_verb = 'POST'
body['parents'] = [parent_id]
body['name'] = path[-1]
# read the first kb to see if it is an empty file
# that is a workaround for https://github.com/kennethreitz/requests/issues/3066
first_chunk = file_obj.read(1024 * 16)
response = self.oauth_session.request(
http_verb,
url,
json=body,
params={
'uploadType': 'resumable',
'fields': FIELDS
},
# headers={'X-Upload-Content-Length': str(size)}
)
response.raise_for_status()
upload_url = response.headers['Location']
if not first_chunk:
response = self.oauth_session.put(upload_url)
else:
chunker = FragmentingChunker(file_obj,
first_chunk=first_chunk,
chunk_size=1024)
headers = {}
if size:
headers['Content-Range'] = 'bytes 0-{}/{}'.format(
size - 1, size)
response = self.oauth_session.put(upload_url,
data=iter(chunker),
headers=headers)
response.raise_for_status()
new_file = response.json()
with jars.TreeToSyncEngineEngineAdapter(node=self.model,
storage_id=self.storage_id,
sync_engine=self._event_sink):
props = gdrive_to_node_props(new_file)
if parent.has_child(new_file['name']):
parent.get_node([new_file['name']]).props.update(props)
else:
parent.add_child(new_file['name'], props)
return new_file[VERSION_FIELD]
@pytest.mark.parametrize(
'exec_func_side_effect, exec_task_state',
[
# INVALID_OPERATION
(BaseException('Boom!'), cc.synctask.SyncTask.INVALID_OPERATION),
(jars.UnavailableError('Boom!',
None), cc.synctask.SyncTask.INVALID_OPERATION),
(jars.InvalidOperationError('Boom!'),
cc.synctask.SyncTask.INVALID_OPERATION),
(FileNotFoundError('Boom!'), cc.synctask.SyncTask.INVALID_OPERATION),
# INVALID_AUTHENTICATION
(jars.AuthenticationError('Boom!'),
cc.synctask.SyncTask.INVALID_AUTHENTICATION),
# VERSION_ID_MISMATCH
(jars.VersionIdNotMatchingError('Boom!'),
cc.synctask.SyncTask.VERSION_ID_MISMATCH),
# CANCELLED
(SyncTaskCancelledException('Boom!'), cc.synctask.SyncTask.CANCELLED),
(jars.CancelledException('Boom!'), cc.synctask.SyncTask.CANCELLED),
])
@pytest.mark.parametrize(
'task', [(cc.synctask.CreateDirSyncTask(None, None, None)),
(cc.synctask.DownloadSyncTask(None, None, None)),
(cc.synctask.UploadSyncTask(None, None, None)),
(cc.synctask.DeleteSyncTask(None, None, None)),
(cc.synctask.FetchFileTreeTask(None, None)),
(cc.synctask.MoveSyncTask(None, None, None, None, None)),
(cc.synctask.CompareSyncTask(None, None))])
def test_worker_run_and_ensure_proper_dispatch(task, exec_func_side_effect,
exec_task_state):
