async def validate_v1_path(self, path, **kwargs): if path == '/': return WaterButlerPath('/', _ids=[self.folder]) obj_id = path.strip('/') files_or_folders = 'folders' if path.endswith('/') else 'files' # Box file ids must be a valid base10 number if not obj_id.isdecimal(): raise exceptions.NotFoundError(str(path)) response = await self.make_request( 'get', self.build_url(files_or_folders, obj_id, fields='id,name,path_collection'), expects=(200, 404,), throws=exceptions.MetadataError, ) if response.status == 404: raise exceptions.NotFoundError(str(path)) data = await response.json() names, ids = zip(*[ (x['name'], x['id']) for x in data['path_collection']['entries'] + [data] ]) names, ids = ('',) + names[ids.index(self.folder) + 1:], ids[ids.index(self.folder):] return WaterButlerPath('/'.join(names), _ids=ids, folder=path.endswith('/'))
async def _metadata_file(self, path: GitLabPath) -> GitLabFileMetadata: """Fetch metadata for the file at ``path`` and build a `GitLabFileMetadata` object for it. The commits endpoint used here will 500 when given a unicode filename. Bug reported here: https://gitlab.com/gitlab-org/gitlab-ce/issues/40776 The `revisions` method uses the same endpoint and will probably encounter the same issue. :param GitLabPath path: the file to get metadata for :rtype: `GitLabFileMetadata` """ file_contents = await self._fetch_file_contents(path) # go to commit history to get modified and created dates last_commit, first_commit, page_nbr = None, None, 1 while page_nbr: url = self._build_repo_url('repository', 'commits', path=path.path, ref_name=path.ref, page=page_nbr, per_page=self.MAX_PAGE_SIZE) logger.debug('file metadata commit history url: {}'.format(url)) resp = await self.make_request( 'GET', url, expects=(200, 404, 500), throws=exceptions.NotFoundError, ) if resp.status == 404: await resp.release() raise exceptions.NotFoundError(path.full_path) if resp.status == 500: # GitLab API is buggy for unicode filenames. Affected files will still work, but # but will have empty created/modified dates. See method docstring await resp.release() break data_page = await resp.json() # GitLab currently returns 200 OK for nonexistent directories # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/34016 # Fallback: empty directories shouldn't exist in git, unless it's the root if page_nbr == 1 and len(data_page) == 0 and not path.is_root: raise exceptions.NotFoundError(path.full_path) if page_nbr == 1: last_commit = data_page[0] first_commit = data_page[-1] page_nbr = resp.headers.get('X-Next-Page', None) file_name = file_contents['file_name'] data = {'name': file_name, 'id': file_contents['blob_id'], 'path': file_contents['file_path'], 'size': file_contents['size'], 'mime_type': mimetypes.guess_type(file_name)[0]} if last_commit is not None: data['modified'] = last_commit['committed_date'] if first_commit is not None: data['created'] = first_commit['committed_date'] return GitLabFileMetadata(data, path, host=self.VIEW_URL, owner=self.owner, repo=self.repo)
async def validate_v1_path(self, path, **kwargs): if not os.path.exists(self.folder + path): raise exceptions.NotFoundError(str(path)) implicit_folder = path.endswith('/') explicit_folder = os.path.isdir(self.folder + path) if implicit_folder != explicit_folder: raise exceptions.NotFoundError(str(path)) return WaterButlerPath(path, prepend=self.folder)
async def delete(self, path, confirm_delete=0, **kwargs): """Delete the entity at ``path``. :param FigsharePath path: Path to be deleted :param int confirm_delete: Must be 1 to confirm root folder delete :rtype: None :raises: :class:`waterbutler.core.exceptions.NotFoundError` :raises: :class:`waterbutler.core.exceptions.DeleteError` Quirks: * If the FigsharePath given is for the provider root path, then the contents of the provider root path will be deleted, but not the provider root itself. """ if not path.identifier: raise exceptions.NotFoundError(str(path)) if path.is_root: if confirm_delete == 1: return await self._delete_container_contents() raise exceptions.DeleteError( 'confirm_delete=1 is required for deleting root provider folder', code=400) if len(path.parts) == 2: if not path.is_folder: raise exceptions.NotFoundError(str(path)) delete_path = (*self.root_path_parts, 'articles', path.parts[1]._id) elif len(path.parts) == 3: if path.is_folder: raise exceptions.NotFoundError(str(path)) article_response = await self.make_request( 'GET', self.build_url(False, *self.root_path_parts, 'articles', path.parts[1]._id), expects=(200, ), ) article_json = await article_response.json() if article_json['defined_type'] in settings.FOLDER_TYPES: delete_path = ('articles', path.parts[1]._id, 'files', path.parts[2]._id) else: delete_path = (*self.root_path_parts, 'articles', path.parts[1]._id) delete_article_response = await self.make_request( 'DELETE', self.build_url(False, *delete_path), expects=(204, ), ) await delete_article_response.release()
async def metadata(self, path, raw=False, folder=False, revision=None, **kwargs): if path.identifier is None: raise exceptions.NotFoundError(str(path)) if path.is_file: return await self._get_file_meta(path, revision=revision, raw=raw) return await self._get_folder_meta(path, raw=raw, folder=folder)
def download(self, path, version=None, revision=None, mode=None, **kwargs): if not path.identifier: raise exceptions.NotFoundError(str(path)) if version is None: # TODO Clean this up # version could be 0 here version = revision # osf storage metadata will return a virtual path within the provider resp = yield from self.make_signed_request( 'GET', self.build_url(path.identifier, 'download', version=version, mode=mode), expects=(200, ), throws=exceptions.DownloadError, ) data = yield from resp.json() provider = self.make_provider(data['settings']) name = data['data'].pop('name') data['data']['path'] = yield from provider.validate_path('/' + data['data']['path']) download_kwargs = {} download_kwargs.update(kwargs) download_kwargs.update(data['data']) download_kwargs['displayName'] = kwargs.get('displayName', name) return (yield from provider.download(**download_kwargs))
async def _delete_folder_contents(self, path: wb_path.WaterButlerPath) -> None: """Given a WaterButlerPath, delete all contents of folder :param WaterButlerPath path: Folder to be emptied :rtype: None :raises: :class:`waterbutler.core.exceptions.NotFoundError` :raises: :class:`waterbutler.core.exceptions.MetadataError` :raises: :class:`waterbutler.core.exceptions.DeleteError` """ file_id = path.identifier if not file_id: raise exceptions.NotFoundError(str(path)) resp = await self.make_request( 'GET', self.build_url('files', q="'{}' in parents".format(file_id), fields='items(id)'), expects=(200, ), throws=exceptions.MetadataError) try: child_ids = (await resp.json())['items'] except (KeyError, IndexError): raise exceptions.MetadataError('{} not found'.format(str(path)), code=HTTPStatus.NOT_FOUND) for child in child_ids: await self.make_request( 'PUT', self.build_url('files', child['id']), data=json.dumps({'labels': {'trashed': 'true'}}), headers={'Content-Type': 'application/json'}, expects=(200, ), throws=exceptions.DeleteError)
def validate_v1_path(self, path, **kwargs): if not getattr(self, '_repo', None): self._repo = yield from self._fetch_repo() self.default_branch = self._repo['default_branch'] branch_ref = kwargs.get('ref') or kwargs.get( 'branch') or self.default_branch implicit_folder = path.endswith('/') url = furl.furl(self.build_repo_url('contents', path)) url.args.update({'ref': branch_ref}) resp = yield from self.make_request('GET', url.url, expects=(200, ), throws=exceptions.MetadataError) content = yield from resp.json() explicit_folder = isinstance(content, list) if implicit_folder != explicit_folder: raise exceptions.NotFoundError(path) path = GitHubPath(path) for part in path.parts: part._id = (branch_ref, None) # TODO Validate that filesha is a valid sha path.parts[-1]._id = (branch_ref, kwargs.get('fileSha')) return path
async def download( self, path: FigsharePath, # type: ignore range: Tuple[int, int] = None, **kwargs) -> streams.ResponseStreamReader: """Download the file identified by ``path`` from this project. :param FigsharePath path: FigsharePath to file you want to download :rtype streams.ResponseStreamReader: """ if not path.is_file: raise exceptions.NotFoundError(str(path)) file_metadata = await self.metadata(path) download_url = file_metadata.extra['downloadUrl'] # type: ignore if download_url is None: raise exceptions.DownloadError('Download not available', code=HTTPStatus.FORBIDDEN) logger.debug('requested-range:: {}'.format(range)) params = {} if file_metadata.is_public else { 'token': self.token } # type: ignore resp = await self.make_request( 'GET', download_url, range=range, params=params, ) if resp.status == 404: await resp.release() raise exceptions.DownloadError('Download not available', code=HTTPStatus.FORBIDDEN) return streams.ResponseStreamReader(resp)
def delete(self, path, **kwargs): if path.identifier is None: raise exceptions.NotFoundError(str(path)) yield from self.make_signed_request('DELETE', self.build_url(path.identifier), params={'user': self.auth['id']}, expects=(200, ))
def test_download_not_found(self): self.mock_provider.download = utils.MockCoroutine( side_effect=exceptions.NotFoundError('/freddie.png')) with pytest.raises(httpclient.HTTPError) as exc: yield self.http_client.fetch( self.get_url('/file?provider=queenhub&path=/freddie.png'), ) assert exc.value.code == 404
async def metadata(self, path, **kwargs): if path.is_root: return (await self._project_metadata_contents()) if path.identifier is None: raise exceptions.NotFoundError(str(path)) provider = await self._make_article_provider(path.parts[1].identifier) return (await provider.metadata(path, **kwargs))
async def validate_v1_path(self, path: str, **kwargs) -> BitbucketPath: commit_sha = kwargs.get('commitSha') branch_name = kwargs.get('branch') # revision query param could be commit sha OR branch # take a guess which one it will be. revision = kwargs.get('revision', None) if revision is not None: try: int(revision, 16) # is revision valid hex? except (TypeError, ValueError): branch_name = revision else: commit_sha = revision if not commit_sha and not branch_name: branch_name = await self._fetch_default_branch() if path == '/': return BitbucketPath(path, _ids=[(commit_sha, branch_name)]) path_obj = BitbucketPath(path) for part in path_obj.parts: part._id = (commit_sha, branch_name) self._parent_dir = await self._fetch_dir_listing(path_obj.parent) if path_obj.is_dir: if path_obj.name not in self._parent_dir['directories']: raise exceptions.NotFoundError(str(path)) else: if path_obj.name not in [ self.bitbucket_path_to_name(x['path'], self._parent_dir['path']) for x in self._parent_dir['files'] ]: raise exceptions.NotFoundError(str(path)) # _fetch_dir_listing will tell us the commit sha used to look up the listing # if not set in path_obj or if the lookup sha is shorter than the returned sha, update it if not commit_sha or (len(commit_sha) < len(self._parent_dir['node'])): path_obj.set_commit_sha(self._parent_dir['node']) return path_obj