def delete(self, path, original_version_id=None): """Deletes a file or directory from onedrive. Ondrive first checks if the version_id, which makes sense, but our tests require us to throw a 404. In order to achieve this, the metadata of the item is fetched first. """ # This will make a call to the api, which will throw 404 if the item is # no longer there. item_meta = self.get_item_meta(path) if original_version_id is not None and original_version_id != 'is_dir': # since the If-Match header are not working reliable we use the # data from the response for checking if there is a conflict try: version_id = self.version_id_from_meta(item_meta) except KeyError: # Raise if the meta does not contain the required keys to return a vid. # TODO: is it better to just return None in version_id_from_meta? raise jars.VersionIdNotMatchingError( version_a=original_version_id, version_b=None) if version_id != original_version_id: raise jars.VersionIdNotMatchingError( version_a=original_version_id, version_b=version_id) drive_id, item_id = self.api.extract_ids(item_meta) endpoint = item_endpoint(drive_id=drive_id, item_id=item_id) self.api.delete(endpoint=endpoint, status_check=raise_for_onedrive)
def move(self, source, target, expected_source_vid=None, expected_target_vid=None): """https://dev.onedrive.com/items/move.htm """ # First handle the source source_meta = self.get_item_meta(source) if expected_source_vid is not None and expected_source_vid != 'is_dir': # check it here if self.version_id_from_meta(source_meta) != expected_source_vid: raise jars.VersionIdNotMatchingError(self.storage_id, expected_source_vid) # Then handle the target target_dir_meta = self.get_item_meta(target[:-1], safe=True) target_drive_id, target_item_id = self.api.extract_ids(target_dir_meta) target_children = self.api.get_children(drive_id=target_drive_id, item_id=target_item_id) for child in target_children['value']: if child['name'] == target[-1]: target_meta = child break if expected_target_vid is not None and expected_target_vid != 'is_dir': if self.version_id_from_meta(target_meta) != expected_target_vid: raise jars.VersionIdNotMatchingError(self.storage_id, expected_target_vid) path_params = { 'parentReference': { 'id': target_item_id, 'driveId': target_drive_id, }, 'name': target[-1], '@name.conflictBehavior': 'replace' } source_drive_id, source_item_id = self.api.extract_ids(source_meta) source_endpoint = item_endpoint(drive_id=source_drive_id, item_id=source_item_id) item_meta = self.api.patch(endpoint=source_endpoint, json=path_params, status_check=raise_for_onedrive).json() if is_dir(item_meta): return jars.IS_DIR else: return self.version_id_from_meta(item_meta)
def delete(self, path, original_version_id): """ Delete can be done in 2 ways on GoogleDrive: either with a permanent delete by using https://developers.google.com/drive/v3/reference/files/delete or set trashed to true with a patch https://developers.google.com/drive/v3/reference/files/update. """ file_id = self._path_to_file_id(path) file_url = urllib.parse.urljoin(self.base_url, 'files/' + str(file_id)) # fetch current version id since Google Drive has node If-Match header if original_version_id is not None: version_id_resp = self.oauth_session.get( file_url, params={'fields': 'mimeType,' + VERSION_FIELD}) version_id_resp.raise_for_status() response = version_id_resp.json() if response['mimeType'] == 'application/vnd.google-apps.folder': version_id = 'is_dir' else: version_id = response[VERSION_FIELD] if version_id != original_version_id: raise jars.VersionIdNotMatchingError( self.storage_id, version_a=version_id, version_b=original_version_id) resp = self.oauth_session.patch(file_url, json={'trashed': True}) resp.raise_for_status()
def raise_for_onedrive(response): """Raise a VersionIdNotMatchingErrorConvert if server returns `if-match` or `if=none-match`.""" # if the format of the version id is not correct the server throws a 400 if response.status_code == 400: if 'if-match' in response.text.lower() or \ 'if-none-match' in response.text.lower(): raise jars.VersionIdNotMatchingError('') raise_for_status(response)
def _verify_version_id(self, path, version_id): current_meta = self._get_meta(path) if version_id == jars.FOLDER_VERSION_ID: return current_meta current_version_id = self._version_id_from_meta(current_meta) if current_version_id != version_id: raise jars.VersionIdNotMatchingError(storage_id=self.storage_id, version_a=version_id, version_b=current_version_id) return current_meta
def _check_version_id(self, file_id, expected_version_id, path=None): url = urllib.parse.urljoin(self.base_url, 'files/{}'.format(file_id)) if expected_version_id is not None: response = self.oauth_session.get( url, params={'fields': 'mimeType,' + VERSION_FIELD}) if expected_version_id == 'is_dir' \ and response.json()['mimeType'] == MIMETYPE_FOLDER: # its fine, dir to dir return response.raise_for_status() if response.json()[VERSION_FIELD] != expected_version_id: raise jars.VersionIdNotMatchingError( self.storage_id, path=path, version_b=expected_version_id, version_a=response.json()[VERSION_FIELD])
def write(self, path, file_obj, original_version_id=None, size=None): """https://dev.onedrive.com/items/upload.htm """ if size is None: raise ValueError('OneDrive requires a size to write') # seperate path into filename and parent_path filename = path[-1] parent_path = path[:-1] # Extract the drive_id and parent_id required to create an item if original_version_id: path_meta_list = list(self.api.iter_path(path)) metadata = path_meta_list[-1] if self.version_id_from_meta(metadata) != original_version_id: raise jars.VersionIdNotMatchingError(self.storage_id, original_version_id) parent_meta = path_meta_list[-2] drive_id, parent_id = self.api.extract_ids(parent_meta) else: # Is the relevant information in the model ? try: parent_node = self.model.get_node(parent_path) drive_id = parent_node.props['_drive_id'] parent_id = parent_node.props['_id'] except KeyError: # The information is not in the model, try the api. parent_meta_list = list(self.api.iter_dir_safe(parent_path)) # The last meta in the list is the parent parent_meta = parent_meta_list[-1] drive_id, parent_id = self.api.extract_ids(parent_meta) response = self.api.upload(drive_id=drive_id, parent_id=parent_id, filename=filename, file_obj=file_obj, size=size) return self.version_id_from_meta(response.json())
def open_read(self, path, expected_version_id=None): """Downloads the file as seen from a filesystems perspective. Try to extract the information required information from the tree, otherwise fall back to the server. https://dev.onedrive.com/items/download.htm """ metadata = self.get_item_meta(path) # Check that the version id is correct if expected_version_id is not None: # headers['If-Match'] = expected_version_id if self.version_id_from_meta(metadata) != expected_version_id: raise jars.VersionIdNotMatchingError(self.storage_id, expected_version_id) # download using the information provided in the metadata. drive_id, item_id = self.api.extract_ids(metadata) return self.api.download(drive_id=drive_id, item_id=item_id)
def open_read(self, path, expected_version_id=None): """Download the file to the filesystem. :param path: a path represented by a list of strings :param expected_version_id :returns: file like object representing of the given file on storage. See :py:class:~`BasicStorage` for full documentation. """ url = self.urls.download(path) headers = {'If-Match': expected_version_id} try: response = self.api.download(url, headers=headers) except HTTPError as error: if error.response.status_code == 404: raise FileNotFoundError elif error.response.status_code == 412: raise jars.VersionIdNotMatchingError() else: raise error return response.raw
def write(self, path, file_obj, original_version_id=None, size=0): """ https://developers.google.com/drive/v3/web/manage-uploads """ # pylint: disable=too-many-locals,too-many-statements parent_id = None node_id = None http_verb = '' with self.model.lock: with contextlib.suppress(KeyError): parent = self.model.get_node(path[:-1]) parent_id = parent.props['_id'] with contextlib.suppress(KeyError): node = parent.get_node([path[-1]]) node_id = node.props['_id'] # create parent and fetch id if parent_id is None: self.make_dir(path[:-1]) with self.model.lock: parent = self.model.get_node(path[:-1]) parent_id = parent.props['_id'] url = 'https://www.googleapis.com/upload/drive/v3/files' # check version id, if there is one to check if original_version_id is not None: res = self.oauth_session.get( 'https://www.googleapis.com/drive/v3/files/{}'.format(node_id), params={'fields': VERSION_FIELD}) res.raise_for_status() if res.json()[VERSION_FIELD] != original_version_id: raise jars.VersionIdNotMatchingError( self.storage_id, version_a=res.json()[VERSION_FIELD], version_b=original_version_id) body = {} # 'parents': [parent_id], 'name': path[-1]} if node_id is not None: http_verb = 'PATCH' body['fileId'] = node_id url += '/' + node_id else: http_verb = 'POST' body['parents'] = [parent_id] body['name'] = path[-1] # read the first kb to see if it is an empty file # that is a workaround for https://github.com/kennethreitz/requests/issues/3066 first_chunk = file_obj.read(1024 * 16) response = self.oauth_session.request( http_verb, url, json=body, params={ 'uploadType': 'resumable', 'fields': FIELDS }, # headers={'X-Upload-Content-Length': str(size)} ) response.raise_for_status() upload_url = response.headers['Location'] if not first_chunk: response = self.oauth_session.put(upload_url) else: chunker = FragmentingChunker(file_obj, first_chunk=first_chunk, chunk_size=1024) headers = {} if size: headers['Content-Range'] = 'bytes 0-{}/{}'.format( size - 1, size) response = self.oauth_session.put(upload_url, data=iter(chunker), headers=headers) response.raise_for_status() new_file = response.json() with jars.TreeToSyncEngineEngineAdapter(node=self.model, storage_id=self.storage_id, sync_engine=self._event_sink): props = gdrive_to_node_props(new_file) if parent.has_child(new_file['name']): parent.get_node([new_file['name']]).props.update(props) else: parent.add_child(new_file['name'], props) return new_file[VERSION_FIELD]
@pytest.mark.parametrize( 'exec_func_side_effect, exec_task_state', [ # INVALID_OPERATION (BaseException('Boom!'), cc.synctask.SyncTask.INVALID_OPERATION), (jars.UnavailableError('Boom!', None), cc.synctask.SyncTask.INVALID_OPERATION), (jars.InvalidOperationError('Boom!'), cc.synctask.SyncTask.INVALID_OPERATION), (FileNotFoundError('Boom!'), cc.synctask.SyncTask.INVALID_OPERATION), # INVALID_AUTHENTICATION (jars.AuthenticationError('Boom!'), cc.synctask.SyncTask.INVALID_AUTHENTICATION), # VERSION_ID_MISMATCH (jars.VersionIdNotMatchingError('Boom!'), cc.synctask.SyncTask.VERSION_ID_MISMATCH), # CANCELLED (SyncTaskCancelledException('Boom!'), cc.synctask.SyncTask.CANCELLED), (jars.CancelledException('Boom!'), cc.synctask.SyncTask.CANCELLED), ]) @pytest.mark.parametrize( 'task', [(cc.synctask.CreateDirSyncTask(None, None, None)), (cc.synctask.DownloadSyncTask(None, None, None)), (cc.synctask.UploadSyncTask(None, None, None)), (cc.synctask.DeleteSyncTask(None, None, None)), (cc.synctask.FetchFileTreeTask(None, None)), (cc.synctask.MoveSyncTask(None, None, None, None, None)), (cc.synctask.CompareSyncTask(None, None))]) def test_worker_run_and_ensure_proper_dispatch(task, exec_func_side_effect, exec_task_state):