async def intra_move(self, # type: ignore dest_provider: provider.BaseProvider, src_path: wb_path.WaterButlerPath, dest_path: wb_path.WaterButlerPath) \ -> typing.Tuple[BaseGoogleDriveMetadata, bool]: self.metrics.add('intra_move.destination_exists', dest_path.identifier is not None) if dest_path.identifier: await dest_provider.delete(dest_path) async with self.request( 'PATCH', self.build_url('files', src_path.identifier), headers={ 'Content-Type': 'application/json' }, data=json.dumps({ 'parents': [{ 'id': dest_path.parent.identifier }], 'title': dest_path.name }), expects=(200, ), throws=exceptions.IntraMoveError, ) as resp: data = await resp.json() created = dest_path.identifier is None dest_path.parts[-1]._id = data['id'] if dest_path.is_dir: metadata = GoogleDriveFolderMetadata(data, dest_path) metadata._children = await self._folder_metadata(dest_path) return metadata, created else: return GoogleDriveFileMetadata(data, dest_path), created # type: ignore
def test_upload_doesnt_unquote(self, provider, file_stream): upload_id = '7' item = fixtures.list_file['items'][0] path = GoogleDrivePath('/birdie%2F %20".jpg', _ids=(provider.folder['id'], None)) start_upload_url = provider._build_upload_url('files', uploadType='resumable') finish_upload_url = provider._build_upload_url('files', uploadType='resumable', upload_id=upload_id) aiohttpretty.register_json_uri('PUT', finish_upload_url, body=item) aiohttpretty.register_uri( 'POST', start_upload_url, headers={ 'LOCATION': 'http://waterbutler.io?upload_id={}'.format(upload_id) }) result, created = yield from provider.upload(file_stream, path) expected = GoogleDriveFileMetadata(item, path).serialized() assert created is True assert result == expected assert aiohttpretty.has_call(method='POST', uri=start_upload_url) assert aiohttpretty.has_call(method='PUT', uri=finish_upload_url)
async def test_upload_update(self, provider, file_stream): upload_id = '7' item = fixtures.list_file['items'][0] path = WaterButlerPath('/birdie.jpg', _ids=(provider.folder['id'], item['id'])) start_upload_url = provider._build_upload_url('files', path.identifier, uploadType='resumable') finish_upload_url = provider._build_upload_url('files', path.identifier, uploadType='resumable', upload_id=upload_id) aiohttpretty.register_json_uri('PUT', finish_upload_url, body=item) aiohttpretty.register_uri( 'PUT', start_upload_url, headers={ 'LOCATION': 'http://waterbutler.io?upload_id={}'.format(upload_id) }) result, created = await provider.upload(file_stream, path) assert aiohttpretty.has_call(method='PUT', uri=start_upload_url) assert aiohttpretty.has_call(method='PUT', uri=finish_upload_url) assert created is False expected = GoogleDriveFileMetadata(item, path) assert result == expected
async def test_upload_create_nested(self, provider, file_stream): upload_id = '7' item = fixtures.list_file['items'][0] path = WaterButlerPath('/ed/sullivan/show.mp3', _ids=[str(x) for x in range(3)]) start_upload_url = provider._build_upload_url('files', uploadType='resumable') finish_upload_url = provider._build_upload_url('files', uploadType='resumable', upload_id=upload_id) aiohttpretty.register_uri( 'POST', start_upload_url, headers={ 'LOCATION': 'http://waterbutler.io?upload_id={}'.format(upload_id) }) aiohttpretty.register_json_uri('PUT', finish_upload_url, body=item) result, created = await provider.upload(file_stream, path) assert aiohttpretty.has_call(method='POST', uri=start_upload_url) assert aiohttpretty.has_call(method='PUT', uri=finish_upload_url) assert created is True expected = GoogleDriveFileMetadata(item, path) assert result == expected
def test_file_metadata_drive_slashes(self, basepath, root_provider_fixtures): item = root_provider_fixtures['file_forward_slash'] path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.provider == 'googledrive' assert parsed.id == item['id'] assert parsed.name == item['title'] assert parsed.name == path.name assert parsed.size == item['fileSize'] assert parsed.size_as_int == 918668 assert type(parsed.size_as_int) == int assert parsed.modified == item['modifiedDate'] assert parsed.content_type == item['mimeType'] assert parsed.extra == { 'revisionId': item['version'], 'webView': item['alternateLink'], 'hashes': { 'md5': item['md5Checksum'] }, } assert parsed.path == '/' + os.path.join(*[x.raw for x in path.parts]) assert parsed.materialized_path == str(path) assert parsed.is_google_doc is False assert parsed.export_name == item['title']
async def intra_copy(self, dest_provider: provider.BaseProvider, src_path: wb_path.WaterButlerPath, dest_path: wb_path.WaterButlerPath) \ -> typing.Tuple[GoogleDriveFileMetadata, bool]: self.metrics.add('intra_copy.destination_exists', dest_path.identifier is not None) if dest_path.identifier: await dest_provider.delete(dest_path) async with self.request( 'POST', self.build_url('files', src_path.identifier, 'copy'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'parents': [{ 'id': dest_path.parent.identifier }], 'title': dest_path.name }), expects=(200, ), throws=exceptions.IntraMoveError, ) as resp: data = await resp.json() # GoogleDrive doesn't support intra-copy for folders, so dest_path will always # be a file. See can_intra_copy() for type check. return GoogleDriveFileMetadata(data, dest_path), dest_path.identifier is None
def _serialize_item(self, path: wb_path.WaterButlerPath, item: dict, raw: bool=False) -> typing.Union[BaseGoogleDriveMetadata, dict]: if raw: return item if item['mimeType'] == self.FOLDER_MIME_TYPE: return GoogleDriveFolderMetadata(item, path) return GoogleDriveFileMetadata(item, path)
def test_metadata_file_root(self, provider): path = WaterButlerPath('/birdie.jpg', _ids=(provider.folder['id'], fixtures.list_file['items'][0]['id'])) list_file_url = provider.build_url('files', path.identifier) aiohttpretty.register_json_uri('GET', list_file_url, body=fixtures.list_file['items'][0]) result = yield from provider.metadata(path) expected = GoogleDriveFileMetadata(fixtures.list_file['items'][0], path) assert result == expected
def test_file_metadata_docs(basepath): item = fixtures.docs_file_metadata path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.name == item['title'] + '.gdoc' assert parsed.extra == { 'revisionId': item['version'], 'downloadExt': '.docx' }
def test_file_metadata_docs(basepath): item = fixtures.docs_file_metadata path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.name == item['title'] + '.gdoc' assert parsed.extra == { 'revisionId': item['version'], 'downloadExt': '.docx', 'webView': item['alternateLink'] } assert parsed.is_google_doc == True assert parsed.export_name == item['title'] + '.docx'
def test_file_metadata_docs(self, basepath, root_provider_fixtures): item = root_provider_fixtures['docs_file_metadata'] path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.name == item['title'] + '.gdoc' assert parsed.extra == { 'revisionId': item['version'], 'downloadExt': '.docx', 'webView': item['alternateLink'], } assert parsed.is_google_doc is True assert parsed.export_name == item['title'] + '.docx'
def test_metadata_root_folder(self, provider): path = yield from provider.validate_path('/') query = provider._build_query(provider.folder['id']) list_file_url = provider.build_url('files', q=query, alt='json') aiohttpretty.register_json_uri('GET', list_file_url, body=fixtures.list_file) result = yield from provider.metadata(path) expected = GoogleDriveFileMetadata( fixtures.list_file['items'][0], path.child(fixtures.list_file['items'][0]['title']) ) assert result == [expected]
async def upload(self, stream, path, **kwargs): assert path.is_file if path.identifier: segments = (path.identifier, ) else: segments = () upload_metadata = self._build_upload_metadata(path.parent.identifier, path.name) upload_id = await self._start_resumable_upload(not path.identifier, segments, stream.size, upload_metadata) data = await self._finish_resumable_upload(segments, stream, upload_id) return GoogleDriveFileMetadata(data, path), path.identifier is None
async def test_metadata_file_nested(self, provider): path = GoogleDrivePath('/hugo/kim/pins', _ids=[str(x) for x in range(4)]) item = fixtures.generate_list(3)['items'][0] url = provider.build_url('files', path.identifier) aiohttpretty.register_json_uri('GET', url, body=item) result = await provider.metadata(path) expected = GoogleDriveFileMetadata(item, path) assert result == expected assert aiohttpretty.has_call(method='GET', uri=url)
def test_file_metadata_drive_slashes(basepath): item = fixtures.file_forward_slash path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.provider == 'googledrive' assert parsed.id == item['id'] assert parsed.name == item['title'] assert parsed.name == path.name assert parsed.size == item['fileSize'] assert parsed.modified == item['modifiedDate'] assert parsed.content_type == item['mimeType'] assert parsed.extra == {'revisionId': item['version']} assert parsed.path == '/' + os.path.join(*[x.raw for x in path.parts]) assert parsed.materialized_path == str(path)
async def test_metadata_folder_nested(self, provider): path = GoogleDrivePath('/hugo/kim/pins/', _ids=[str(x) for x in range(4)]) body = fixtures.generate_list(3) item = body['items'][0] query = provider._build_query(path.identifier) url = provider.build_url('files', q=query, alt='json', maxResults=1000) aiohttpretty.register_json_uri('GET', url, body=body) result = await provider.metadata(path) expected = GoogleDriveFileMetadata(item, path.child(item['title'])) assert result == [expected] assert aiohttpretty.has_call(method='GET', uri=url)
async def intra_copy(self, dest_provider, src_path, dest_path): if dest_path.identifier: await dest_provider.delete(dest_path) async with self.request( 'POST', self.build_url('files', src_path.identifier, 'copy'), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'parents': [{ 'id': dest_path.parent.identifier }], 'title': dest_path.name }), expects=(200, ), throws=exceptions.IntraMoveError, ) as resp: data = await resp.json() return GoogleDriveFileMetadata(data, dest_path), dest_path.identifier is None
def test_file_metadata_drive(basepath): item = fixtures.list_file['items'][0] path = basepath.child(item['title']) parsed = GoogleDriveFileMetadata(item, path) assert parsed.provider == 'googledrive' assert parsed.id == item['id'] assert path.name == item['title'] assert parsed.name == item['title'] assert parsed.size == item['fileSize'] assert parsed.modified == item['modifiedDate'] assert parsed.content_type == item['mimeType'] assert parsed.extra == { 'revisionId': item['version'], 'webView': item['alternateLink'] } assert parsed.path == '/' + os.path.join(*[x.raw for x in path.parts]) assert parsed.materialized_path == str(path) assert parsed.is_google_doc == False assert parsed.export_name == item['title']
async def upload(self, stream, path: wb_path.WaterButlerPath, *args, **kwargs) \ -> typing.Tuple[GoogleDriveFileMetadata, bool]: assert path.is_file if path.identifier: segments = [path.identifier] else: segments = [] stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5)) upload_metadata = self._build_upload_metadata(path.parent.identifier, path.name) upload_id = await self._start_resumable_upload(not path.identifier, segments, stream.size, upload_metadata) data = await self._finish_resumable_upload(segments, stream, upload_id) if data['md5Checksum'] != stream.writers['md5'].hexdigest: raise exceptions.UploadChecksumMismatchError() return GoogleDriveFileMetadata(data, path), path.identifier is None
def intra_move(self, dest_provider, src_path, dest_path): if dest_path.identifier: yield from dest_provider.delete(dest_path) resp = yield from self.make_request( 'PATCH', self.build_url('files', src_path.identifier), headers={'Content-Type': 'application/json'}, data=json.dumps({ 'parents': [{ 'id': dest_path.parent.identifier }], 'title': dest_path.name }), expects=(200, ), throws=exceptions.IntraMoveError, ) data = yield from resp.json() return GoogleDriveFileMetadata(data, dest_path), dest_path.identifier is None
def _serialize_item(self, path, item, raw=False): if raw: return item if item['mimeType'] == self.FOLDER_MIME_TYPE: return GoogleDriveFolderMetadata(item, path) return GoogleDriveFileMetadata(item, path)
def _serialize_item(self, path, item, raw=False): if raw: return item if item['mimeType'] == 'application/vnd.google-apps.folder': return GoogleDriveFolderMetadata(item, path) return GoogleDriveFileMetadata(item, path)
async def _file_metadata(self, path: GoogleDrivePath, revision: str=None, raw: bool=False): """ Returns metadata for the file identified by `path`. If the `revision` arg is set, will attempt to return metadata for the given revision of the file. If the revision does not exist, ``_file_metadata`` will throw a 404. This method used to error with a 500 when metadata was requested for a file that the authorizing user only had view or commenting permissions for. The GDrive revisions endpoint returns a 403, which was not being handled. WB postpends a sentinel value to the revisions for these files. If a revision ending with this sentinel value is detected, this method will return metadata for the latest revision of the file. If a revision NOT ending in the sentinel value is requested for a read-only file, this method will return a 404 Not Found instead. Metrics: ``_file_metadata.got_revision``: did this request include a revision parameter? ``_file_metadata.revision_is_valid``: if a revision was given, was it valid? A revision is "valid" if it doesn't end with our sentinal string (`settings.DRIVE_IGNORE_VERSION`). ``_file_metadata.user_role``: What role did the user possess? Helps identify other roles for which revision information isn't available. :param GoogleDrivePath path: the path of the file whose metadata is being requested :param str revision: a string representing the ID of the revision (default: `None`) :param bool raw: should we return the raw response object from the GDrive API? :rtype: GoogleDriveFileMetadata :rtype: dict :return: a metadata for the googledoc or the raw response object from the GDrive API """ self.metrics.add('_file_metadata.got_revision', revision is not None) valid_revision = revision and not revision.endswith(settings.DRIVE_IGNORE_VERSION) if revision: self.metrics.add('_file_metadata.revision_is_valid', valid_revision) if revision and valid_revision: url = self.build_url('files', path.identifier, 'revisions', revision) else: url = self.build_url('files', path.identifier) async with self.request( 'GET', url, expects=(200, 403, 404, ), throws=exceptions.MetadataError, ) as resp: try: data = await resp.json() except: # some 404s return a string instead of json data = await resp.read() if resp.status != 200: raise exceptions.NotFoundError(path) if revision and valid_revision: return GoogleDriveFileRevisionMetadata(data, path) user_role = data['userPermission']['role'] self.metrics.add('_file_metadata.user_role', user_role) can_access_revisions = user_role in self.ROLES_ALLOWING_REVISIONS if drive_utils.is_docs_file(data): if can_access_revisions: return await self._handle_docs_versioning(path, data, raw=raw) else: # Revisions are not available for some sharing configurations. If revisions list is # empty, use the etag of the file plus a sentinel string as a dummy revision ID. data['version'] = data['etag'] + settings.DRIVE_IGNORE_VERSION return data if raw else GoogleDriveFileMetadata(data, path)