async def validate_v1_path(self, path, **kwargs):
if path == '/':
return WaterButlerPath('/', _ids=[self.folder])
obj_id = path.strip('/')
files_or_folders = 'folders' if path.endswith('/') else 'files'
# Box file ids must be a valid base10 number
if not obj_id.isdecimal():
raise exceptions.NotFoundError(str(path))
response = await self.make_request(
'get',
self.build_url(files_or_folders, obj_id, fields='id,name,path_collection'),
expects=(200, 404,),
throws=exceptions.MetadataError,
)
if response.status == 404:
raise exceptions.NotFoundError(str(path))
data = await response.json()
names, ids = zip(*[
(x['name'], x['id'])
for x in
data['path_collection']['entries'] + [data]
])
names, ids = ('',) + names[ids.index(self.folder) + 1:], ids[ids.index(self.folder):]
return WaterButlerPath('/'.join(names), _ids=ids, folder=path.endswith('/'))
async def _metadata_file(self, path: GitLabPath) -> GitLabFileMetadata:
"""Fetch metadata for the file at ``path`` and build a `GitLabFileMetadata` object for it.
The commits endpoint used here will 500 when given a unicode filename. Bug reported here:
https://gitlab.com/gitlab-org/gitlab-ce/issues/40776 The `revisions` method uses the same
endpoint and will probably encounter the same issue.
:param GitLabPath path: the file to get metadata for
:rtype: `GitLabFileMetadata`
"""
file_contents = await self._fetch_file_contents(path)
# go to commit history to get modified and created dates
last_commit, first_commit, page_nbr = None, None, 1
while page_nbr:
url = self._build_repo_url('repository', 'commits', path=path.path,
ref_name=path.ref, page=page_nbr,
per_page=self.MAX_PAGE_SIZE)
logger.debug('file metadata commit history url: {}'.format(url))
resp = await self.make_request(
'GET',
url,
expects=(200, 404, 500),
throws=exceptions.NotFoundError,
)
if resp.status == 404:
await resp.release()
raise exceptions.NotFoundError(path.full_path)
if resp.status == 500:
# GitLab API is buggy for unicode filenames. Affected files will still work, but
# but will have empty created/modified dates. See method docstring
await resp.release()
break
data_page = await resp.json()
# GitLab currently returns 200 OK for nonexistent directories
# See: https://gitlab.com/gitlab-org/gitlab-ce/issues/34016
# Fallback: empty directories shouldn't exist in git, unless it's the root
if page_nbr == 1 and len(data_page) == 0 and not path.is_root:
raise exceptions.NotFoundError(path.full_path)
if page_nbr == 1:
last_commit = data_page[0]
first_commit = data_page[-1]
page_nbr = resp.headers.get('X-Next-Page', None)
file_name = file_contents['file_name']
data = {'name': file_name, 'id': file_contents['blob_id'],
'path': file_contents['file_path'], 'size': file_contents['size'],
'mime_type': mimetypes.guess_type(file_name)[0]}
if last_commit is not None:
data['modified'] = last_commit['committed_date']
if first_commit is not None:
data['created'] = first_commit['committed_date']
return GitLabFileMetadata(data, path, host=self.VIEW_URL, owner=self.owner, repo=self.repo)
async def validate_v1_path(self, path, **kwargs):
if not os.path.exists(self.folder + path):
raise exceptions.NotFoundError(str(path))
implicit_folder = path.endswith('/')
explicit_folder = os.path.isdir(self.folder + path)
if implicit_folder != explicit_folder:
raise exceptions.NotFoundError(str(path))
return WaterButlerPath(path, prepend=self.folder)
async def delete(self, path, confirm_delete=0, **kwargs):
"""Delete the entity at ``path``.
:param FigsharePath path: Path to be deleted
:param int confirm_delete: Must be 1 to confirm root folder delete
:rtype: None
:raises: :class:`waterbutler.core.exceptions.NotFoundError`
:raises: :class:`waterbutler.core.exceptions.DeleteError`
Quirks:
* If the FigsharePath given is for the provider root path, then the contents of the
provider root path will be deleted, but not the provider root itself.
"""
if not path.identifier:
raise exceptions.NotFoundError(str(path))
if path.is_root:
if confirm_delete == 1:
return await self._delete_container_contents()
raise exceptions.DeleteError(
'confirm_delete=1 is required for deleting root provider folder',
code=400)
if len(path.parts) == 2:
if not path.is_folder:
raise exceptions.NotFoundError(str(path))
delete_path = (*self.root_path_parts, 'articles',
path.parts[1]._id)
elif len(path.parts) == 3:
if path.is_folder:
raise exceptions.NotFoundError(str(path))
article_response = await self.make_request(
'GET',
self.build_url(False, *self.root_path_parts, 'articles',
path.parts[1]._id),
expects=(200, ),
)
article_json = await article_response.json()
if article_json['defined_type'] in settings.FOLDER_TYPES:
delete_path = ('articles', path.parts[1]._id, 'files',
path.parts[2]._id)
else:
delete_path = (*self.root_path_parts, 'articles',
path.parts[1]._id)
delete_article_response = await self.make_request(
'DELETE',
self.build_url(False, *delete_path),
expects=(204, ),
)
await delete_article_response.release()
async def metadata(self, path, raw=False, folder=False, revision=None, **kwargs):
if path.identifier is None:
raise exceptions.NotFoundError(str(path))
if path.is_file:
return await self._get_file_meta(path, revision=revision, raw=raw)
return await self._get_folder_meta(path, raw=raw, folder=folder)
def download(self, path, version=None, revision=None, mode=None, **kwargs):
if not path.identifier:
raise exceptions.NotFoundError(str(path))
if version is None:
# TODO Clean this up
# version could be 0 here
version = revision
# osf storage metadata will return a virtual path within the provider
resp = yield from self.make_signed_request(
'GET',
self.build_url(path.identifier, 'download', version=version, mode=mode),
expects=(200, ),
throws=exceptions.DownloadError,
)
data = yield from resp.json()
provider = self.make_provider(data['settings'])
name = data['data'].pop('name')
data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
download_kwargs = {}
download_kwargs.update(kwargs)
download_kwargs.update(data['data'])
download_kwargs['displayName'] = kwargs.get('displayName', name)
return (yield from provider.download(**download_kwargs))
async def _delete_folder_contents(self, path: wb_path.WaterButlerPath) -> None:
"""Given a WaterButlerPath, delete all contents of folder
:param WaterButlerPath path: Folder to be emptied
:rtype: None
:raises: :class:`waterbutler.core.exceptions.NotFoundError`
:raises: :class:`waterbutler.core.exceptions.MetadataError`
:raises: :class:`waterbutler.core.exceptions.DeleteError`
"""
file_id = path.identifier
if not file_id:
raise exceptions.NotFoundError(str(path))
resp = await self.make_request(
'GET',
self.build_url('files',
q="'{}' in parents".format(file_id),
fields='items(id)'),
expects=(200, ),
throws=exceptions.MetadataError)
try:
child_ids = (await resp.json())['items']
except (KeyError, IndexError):
raise exceptions.MetadataError('{} not found'.format(str(path)),
code=HTTPStatus.NOT_FOUND)
for child in child_ids:
await self.make_request(
'PUT',
self.build_url('files', child['id']),
data=json.dumps({'labels': {'trashed': 'true'}}),
headers={'Content-Type': 'application/json'},
expects=(200, ),
throws=exceptions.DeleteError)
def validate_v1_path(self, path, **kwargs):
if not getattr(self, '_repo', None):
self._repo = yield from self._fetch_repo()
self.default_branch = self._repo['default_branch']
branch_ref = kwargs.get('ref') or kwargs.get(
'branch') or self.default_branch
implicit_folder = path.endswith('/')
url = furl.furl(self.build_repo_url('contents', path))
url.args.update({'ref': branch_ref})
resp = yield from self.make_request('GET',
url.url,
expects=(200, ),
throws=exceptions.MetadataError)
content = yield from resp.json()
explicit_folder = isinstance(content, list)
if implicit_folder != explicit_folder:
raise exceptions.NotFoundError(path)
path = GitHubPath(path)
for part in path.parts:
part._id = (branch_ref, None)
# TODO Validate that filesha is a valid sha
path.parts[-1]._id = (branch_ref, kwargs.get('fileSha'))
return path
async def download(
self,
path: FigsharePath, # type: ignore
range: Tuple[int, int] = None,
**kwargs) -> streams.ResponseStreamReader:
"""Download the file identified by ``path`` from this project.
:param FigsharePath path: FigsharePath to file you want to download
:rtype streams.ResponseStreamReader:
"""
if not path.is_file:
raise exceptions.NotFoundError(str(path))
file_metadata = await self.metadata(path)
download_url = file_metadata.extra['downloadUrl'] # type: ignore
if download_url is None:
raise exceptions.DownloadError('Download not available',
code=HTTPStatus.FORBIDDEN)
logger.debug('requested-range:: {}'.format(range))
params = {} if file_metadata.is_public else {
'token': self.token
} # type: ignore
resp = await self.make_request(
'GET',
download_url,
range=range,
params=params,
)
if resp.status == 404:
await resp.release()
raise exceptions.DownloadError('Download not available',
code=HTTPStatus.FORBIDDEN)
return streams.ResponseStreamReader(resp)
def delete(self, path, **kwargs):
if path.identifier is None:
raise exceptions.NotFoundError(str(path))
yield from self.make_signed_request('DELETE',
self.build_url(path.identifier),
params={'user': self.auth['id']},
expects=(200, ))
def test_download_not_found(self):
self.mock_provider.download = utils.MockCoroutine(
side_effect=exceptions.NotFoundError('/freddie.png'))
with pytest.raises(httpclient.HTTPError) as exc:
yield self.http_client.fetch(
self.get_url('/file?provider=queenhub&path=/freddie.png'), )
assert exc.value.code == 404
async def metadata(self, path, **kwargs):
if path.is_root:
return (await self._project_metadata_contents())
if path.identifier is None:
raise exceptions.NotFoundError(str(path))
provider = await self._make_article_provider(path.parts[1].identifier)
return (await provider.metadata(path, **kwargs))
async def validate_v1_path(self, path: str, **kwargs) -> BitbucketPath:
commit_sha = kwargs.get('commitSha')
branch_name = kwargs.get('branch')
# revision query param could be commit sha OR branch
# take a guess which one it will be.
revision = kwargs.get('revision', None)
if revision is not None:
try:
int(revision, 16) # is revision valid hex?
except (TypeError, ValueError):
branch_name = revision
else:
commit_sha = revision
if not commit_sha and not branch_name:
branch_name = await self._fetch_default_branch()
if path == '/':
return BitbucketPath(path, _ids=[(commit_sha, branch_name)])
path_obj = BitbucketPath(path)
for part in path_obj.parts:
part._id = (commit_sha, branch_name)
self._parent_dir = await self._fetch_dir_listing(path_obj.parent)
if path_obj.is_dir:
if path_obj.name not in self._parent_dir['directories']:
raise exceptions.NotFoundError(str(path))
else:
if path_obj.name not in [
self.bitbucket_path_to_name(x['path'],
self._parent_dir['path'])
for x in self._parent_dir['files']
]:
raise exceptions.NotFoundError(str(path))
# _fetch_dir_listing will tell us the commit sha used to look up the listing
# if not set in path_obj or if the lookup sha is shorter than the returned sha, update it
if not commit_sha or (len(commit_sha) < len(self._parent_dir['node'])):
path_obj.set_commit_sha(self._parent_dir['node'])
return path_obj
