def prevalidate_put(self):
"""Prevalidation for creation requests. Runs BEFORE the body of a request is accepted and
before the path given in the url has been validated. An early rejection here will save us
one or more API calls to the provider. Requests with bodies that are too large can be
rejected if we have not began to accept the body. Validation is as follows:
1. Pull kind from query params. It must be file, folder, or not included (which defaults to file)
2. Ensure that content length is present for file uploads
3. Ensure that content length is either not present or 0 for folder creation requests
"""
self.kind = self.get_query_argument('kind', default='file')
if self.kind not in ('file', 'folder'):
raise exceptions.InvalidParameters(
'Kind must be file, folder or unspecified (interpreted as file), not {}'
.format(self.kind))
length = self.request.headers.get('Content-Length')
if length is None and self.kind == 'file':
# Length Required
raise exceptions.InvalidParameters(
'Content-Length is required for file uploads', code=411)
try:
if length is not None and int(
length) > 0 and self.kind == 'folder':
# Payload Too Large
raise exceptions.InvalidParameters(
'Folder creation requests may not have a body', code=413)
except ValueError:
raise exceptions.InvalidParameters('Invalid Content-Length')
def postvalidate_put(self):
"""Postvalidation for creation requests. Runs BEFORE the body of a request is accepted, but
after the path has been validated. Invalid path+params combinations can be rejected here.
Validation is as follows:
1. If path is a folder, the name parameter must be present.
2. If path is a file, the name parameter must be absent.
3. If the entity being created is a folder, then path must be a folder as well.
"""
self.childs_name = self.get_query_argument('name', default=None)
if self.path.is_dir:
if self.childs_name is None:
raise exceptions.InvalidParameters(
'Missing required parameter \'name\'')
self.target_path = self.path.child(self.childs_name,
folder=(self.kind == 'folder'))
else:
if self.childs_name is not None:
raise exceptions.InvalidParameters(
"'name' parameter doesn't apply to actions on files")
if self.kind == 'folder':
raise exceptions.InvalidParameters(
'Path must be a folder (and end with a "/") if trying to create a subfolder',
code=409)
self.target_path = self.path
async def postvalidate_put(self):
"""Postvalidation for creation requests. Runs BEFORE the body of a request is accepted, but
after the path has been validated. Invalid path+params combinations can be rejected here.
Validation is as follows:
1. If path is a folder, the name parameter must be present.
2. If path is a file, the name parameter must be absent.
3. If the entity being created is a folder, then path must be a folder as well.
Also checks to make sure V1 semantics are being respected. If a PUT request is issued
against a folder then we check to make sure that an entity with the same name does not
already exist there. If it does, then we issue a 409 Conflict. For providers that do not
allow entites of different types to have the same name (e.g. Github), we also check to make
sure that such an entity does not exist.
"""
self.childs_name = self.get_query_argument('name', default=None)
# handle newfile and newfolder naming conflicts
if self.path.is_dir:
if self.childs_name is None:
raise exceptions.InvalidParameters(
'Missing required parameter \'name\'')
self.target_path = self.path.child(self.childs_name,
folder=(self.kind == 'folder'))
# osfstorage, box, and googledrive need ids before calling exists()
validated_target_path = await self.provider.revalidate_path(
self.path, self.target_path.name, self.target_path.is_dir)
my_type_exists = await self.provider.exists(validated_target_path)
if not isinstance(my_type_exists, bool) or my_type_exists:
raise exceptions.NamingConflict(self.target_path)
if not self.provider.can_duplicate_names():
target_flipped = self.path.child(
self.childs_name, folder=(self.kind != 'folder'))
# osfstorage, box, and googledrive need ids before calling exists(), but only box
# disallows can_duplicate_names and needs this.
validated_target_flipped = await self.provider.revalidate_path(
self.path, target_flipped.name, target_flipped.is_dir)
other_exists = await self.provider.exists(
validated_target_flipped)
# the dropbox provider's metadata() method returns a [] here instead of True
if not isinstance(other_exists, bool) or other_exists:
raise exceptions.NamingConflict(self.target_path)
else:
if self.childs_name is not None:
raise exceptions.InvalidParameters(
"'name' parameter doesn't apply to actions on files")
if self.kind == 'folder':
raise exceptions.InvalidParameters(
'Path must be a folder (and end with a "/") if trying to create a subfolder',
code=409)
self.target_path = self.path
def validate_post(self):
try:
if int(self.request.headers['Content-Length']) > 1 * MBs:
# There should be no JSON body > 1 megs
raise exceptions.InvalidParameters(
'Request body must be under 1Mb', code=413)
except (KeyError, ValueError):
raise exceptions.InvalidParameters('Content-Length is required',
code=411)
def prevalidate_post(self):
"""Validate body and query parameters before spending API calls on validating path. We
don't trust path yet, so I don't wanna see it being used here. Current validations:
1. Max body size is 1Mb.
2. Content-Length header must be provided.
"""
try:
if int(self.request.headers['Content-Length']) > 1 * MBs:
# There should be no JSON body > 1 megs
raise exceptions.InvalidParameters('Request body must be under 1Mb', code=413)
except (KeyError, ValueError):
raise exceptions.InvalidParameters('Content-Length is required', code=411)
async def validate_v1_path(self, path, **kwargs):
if not getattr(self, '_repo', None):
self._repo = await self._fetch_repo()
self.default_branch = self._repo['default_branch']
branch_ref = kwargs.get('ref') or kwargs.get(
'branch') or self.default_branch
if isinstance(branch_ref, list):
raise exceptions.InvalidParameters(
'Only one ref or branch may be given.')
if path == '/':
return GitHubPath(path, _ids=[(branch_ref, '')])
branch_data = await self._fetch_branch(branch_ref)
# throws Not Found if path not in tree
await self._search_tree_for_path(
path, branch_data['commit']['commit']['tree']['sha'])
path = GitHubPath(path)
for part in path.parts:
part._id = (branch_ref, None)
# TODO Validate that filesha is a valid sha
path.parts[-1]._id = (branch_ref, kwargs.get('fileSha'))
return path
async def validate_path(self, path, **kwargs):
if not getattr(self, '_repo', None):
self._repo = await self._fetch_repo()
self.default_branch = self._repo['default_branch']
path = GitHubPath(path)
branch_ref, ref_from = None, None
if kwargs.get('ref'):
branch_ref = kwargs.get('ref')
ref_from = 'query_ref'
elif kwargs.get('branch'):
branch_ref = kwargs.get('branch')
ref_from = 'query_branch'
else:
branch_ref = self.default_branch
ref_from = 'default_branch'
if isinstance(branch_ref, list):
raise exceptions.InvalidParameters('Only one ref or branch may be given.')
self.metrics.add('branch_ref_from', ref_from)
for part in path.parts:
part._id = (branch_ref, None)
# TODO Validate that filesha is a valid sha
path.parts[-1]._id = (branch_ref, kwargs.get('fileSha'))
self.metrics.add('file_sha_given', True if kwargs.get('fileSha') else False)
return path
def json(self):
if not hasattr(self, '_json'):
try:
# self.body is defined by self.data_received
self._json = json.loads(self.body.decode())
except ValueError:
raise exceptions.InvalidParameters('Invalid json body')
return self._json
def validate_put(self):
"""Prevalidation for creation requests. Run BEFORE
the body of a request is accepted. Requests with bodies that are too large can be
rejected if we have not began to accept the body.
Validation is as follows:
1. Pull kind from query params. It must be file, folder, or not included (which defaults to file)
2. If path is a folder (ends with a slash) pull name from query parameters raise an exception if its not found
* If kind is folder a / is append to path
3. If path does not end with a / and kind is folder raise an exception
4. Ensure that content length is present for file uploads
5. Ensure that content length is either not present or 0 for folder creation requests
"""
self.kind = self.get_query_argument('kind', default='file')
if self.kind not in ('file', 'folder'):
raise exceptions.InvalidParameters(
'Kind must be file, folder or unspecified (interpreted as file), not {}'
.format(self.kind))
if self.path.endswith('/'):
name = self.get_query_argument('name') # TODO What does this do?
self.path = os.path.join(self.path, name)
if self.kind == 'folder':
self.path += '/'
elif self.kind == 'folder':
raise exceptions.InvalidParameters(
'Path must end with a / if kind is folder')
length = self.request.headers.get('Content-Length')
if length is None and self.kind == 'file':
# Length Required
raise exceptions.InvalidParameters(
'Content-Length is required for file uploads', code=411)
try:
if int(length) > 0 and self.kind == 'folder':
# Payload Too Large
raise exceptions.InvalidParameters(
'Folder creation requests may not have a body', code=413)
except ValueError:
raise exceptions.InvalidParameters('Invalid Content-Length')
async def validate_path(self, path, **kwargs):
if not getattr(self, '_repo', None):
self._repo = await self._fetch_repo()
self.default_branch = self._repo['default_branch']
path = GitHubPath(path)
branch_ref = kwargs.get('ref') or kwargs.get(
'branch') or self.default_branch
if isinstance(branch_ref, list):
raise exceptions.InvalidParameters(
'Only one ref or branch may be given.')
for part in path.parts:
part._id = (branch_ref, None)
# TODO Validate that filesha is a valid sha
path.parts[-1]._id = (branch_ref, kwargs.get('fileSha'))
return path
async def move_or_copy(self):
"""Copy, move, and rename files and folders.
**Auth actions**: ``copy``, ``move``, or ``rename``
**Provider actions**: ``copy`` or ``move``
*Auth actions* come from the ``action`` body parameter in the request and are used by the
auth handler.
*Provider actions* are determined from the *auth action*. A "rename" is a special case of
the "move" provider action that implies that the destination resource, provider, and parent
path will all be the same as the source.
"""
# Force the json body to load into memory
await self.request.body
auth_action = self.json.get('action', 'null')
if auth_action not in ('copy', 'move', 'rename'):
raise exceptions.InvalidParameters('Auth action must be "copy", "move", or "rename", '
'not "{}"'.format(auth_action))
# Provider setup is delayed so the provider action can be updated from the auth action.
provider = self.path_kwargs.get('provider', '')
provider_action = auth_action
if auth_action == 'rename':
if not self.json.get('rename', ''):
raise exceptions.InvalidParameters('"rename" field is required for renaming')
provider_action = 'move'
self.auth = await auth_handler.get(
self.resource,
provider,
self.request,
action=auth_action,
auth_type=AuthType.SOURCE
)
self.provider = make_provider(
provider,
self.auth['auth'],
self.auth['credentials'],
self.auth['settings']
)
self.path = await self.provider.validate_v1_path(self.path, **self.arguments)
if auth_action == 'rename': # 'rename' implies the file/folder does not change location
self.dest_auth = self.auth
self.dest_provider = self.provider
self.dest_path = self.path.parent
self.dest_resource = self.resource
else:
path = self.json.get('path', None)
if path is None:
raise exceptions.InvalidParameters('"path" field is required for moves or copies')
if not path.endswith('/'):
raise exceptions.InvalidParameters(
'"path" field requires a trailing slash to indicate it is a folder'
)
# TODO optimize for same provider and resource
# for copy action, `auth_action` is the same as `provider_action`
if auth_action == 'copy' and self.path.is_root and not self.json.get('rename'):
raise exceptions.InvalidParameters('"rename" field is required for copying root')
# Note: attached to self so that _send_hook has access to these
self.dest_resource = self.json.get('resource', self.resource)
self.dest_auth = await auth_handler.get(
self.dest_resource,
self.json.get('provider', self.provider.NAME),
self.request,
action=auth_action,
auth_type=AuthType.DESTINATION,
)
self.dest_provider = make_provider(
self.json.get('provider', self.provider.NAME),
self.dest_auth['auth'],
self.dest_auth['credentials'],
self.dest_auth['settings']
)
self.dest_path = await self.dest_provider.validate_path(**self.json)
if not getattr(self.provider, 'can_intra_' + provider_action)(self.dest_provider, self.path):
# this weird signature syntax courtesy of py3.4 not liking trailing commas on kwargs
conflict = self.json.get('conflict', DEFAULT_CONFLICT)
result = await getattr(tasks, provider_action).adelay(
rename=self.json.get('rename'),
conflict=conflict,
request=remote_logging._serialize_request(self.request),
*self.build_args()
)
metadata, created = await tasks.wait_on_celery(result)
else:
metadata, created = (
await tasks.backgrounded(
getattr(self.provider, provider_action),
self.dest_provider,
self.path,
self.dest_path,
rename=self.json.get('rename'),
conflict=self.json.get('conflict', DEFAULT_CONFLICT),
)
)
self.dest_meta = metadata
if created:
self.set_status(int(HTTPStatus.CREATED))
else:
self.set_status(int(HTTPStatus.OK))
self.write({'data': metadata.json_api_serialized(self.dest_resource)})
async def move_or_copy(self):
# Force the json body to load into memory
await self.request.body
if self.json.get('action') not in ('copy', 'move', 'rename'):
# Note: null is used as the default to avoid python specific error messages
raise exceptions.InvalidParameters(
'Action must be copy, move or rename, not {}'.format(
self.json.get('action', 'null')))
if self.json['action'] == 'rename':
if not self.json.get('rename'):
raise exceptions.InvalidParameters(
'Rename is required for renaming')
action = 'move'
self.dest_auth = self.auth
self.dest_provider = self.provider
self.dest_path = self.path.parent
self.dest_resource = self.resource
else:
if 'path' not in self.json:
raise exceptions.InvalidParameters(
'Path is required for moves or copies')
action = self.json['action']
# Note: attached to self so that _send_hook has access to these
self.dest_resource = self.json.get('resource', self.resource)
# TODO optimize for same provider and resource
self.dest_auth = await auth_handler.get(
self.dest_resource,
self.json.get('provider', self.provider.NAME), self.request)
self.dest_provider = make_provider(
self.json.get('provider',
self.provider.NAME), self.dest_auth['auth'],
self.dest_auth['credentials'], self.dest_auth['settings'])
self.dest_path = await self.dest_provider.validate_path(
self.json['path'])
if not getattr(self.provider, 'can_intra_' + action)(
self.dest_provider, self.path):
# this weird signature syntax courtesy of py3.4 not liking trailing commas on kwargs
result = await getattr(tasks, action).adelay(
rename=self.json.get('rename'),
conflict=self.json.get('conflict', DEFAULT_CONFLICT),
*self.build_args())
metadata, created = await tasks.wait_on_celery(result)
else:
metadata, created = (await tasks.backgrounded(
getattr(self.provider, action),
self.dest_provider,
self.path,
self.dest_path,
rename=self.json.get('rename'),
conflict=self.json.get('conflict', DEFAULT_CONFLICT),
))
self.dest_meta = metadata
if created:
self.set_status(201)
else:
self.set_status(200)
self.write({'data': metadata.json_api_serialized(self.dest_resource)})
async def move_or_copy(self):
# Force the json body to load into memory
await self.request.body
if self.json.get('action') not in ('copy', 'move', 'rename'):
# Note: null is used as the default to avoid python specific error messages
raise exceptions.InvalidParameters(
'Action must be copy, move or rename, '
'not {}'.format(self.json.get('action', 'null')))
# Setup of the provider was delayed so the json action could be retrieved from the request body.
provider = self.path_kwargs['provider']
action = self.json['action']
self.auth = await auth_handler.get(self.resource,
provider,
self.request,
action=action,
auth_type=AuthType.SOURCE)
self.provider = make_provider(provider, self.auth['auth'],
self.auth['credentials'],
self.auth['settings'])
self.path = await self.provider.validate_v1_path(
self.path, **self.arguments)
if action == 'rename':
if not self.json.get('rename'):
raise exceptions.InvalidParameters(
'Rename is required for renaming')
action = 'move'
self.dest_auth = self.auth
self.dest_provider = self.provider
self.dest_path = self.path.parent
self.dest_resource = self.resource
else:
if 'path' not in self.json:
raise exceptions.InvalidParameters(
'Path is required for moves or copies')
if not self.json['path'].endswith('/'):
raise exceptions.InvalidParameters(
'Path requires a trailing slash to indicate '
'it is a folder')
# TODO optimize for same provider and resource
# Note: attached to self so that _send_hook has access to these
self.dest_resource = self.json.get('resource', self.resource)
self.dest_auth = await auth_handler.get(
self.dest_resource,
self.json.get('provider', self.provider.NAME),
self.request,
action=action,
auth_type=AuthType.DESTINATION,
)
self.dest_provider = make_provider(
self.json.get('provider',
self.provider.NAME), self.dest_auth['auth'],
self.dest_auth['credentials'], self.dest_auth['settings'])
self.dest_path = await self.dest_provider.validate_path(**self.json
)
if not getattr(self.provider, 'can_intra_' + action)(
self.dest_provider, self.path):
# this weird signature syntax courtesy of py3.4 not liking trailing commas on kwargs
conflict = self.json.get('conflict', DEFAULT_CONFLICT)
result = await getattr(tasks, action).adelay(
rename=self.json.get('rename'),
conflict=conflict,
request=remote_logging._serialize_request(self.request),
*self.build_args())
metadata, created = await tasks.wait_on_celery(result)
else:
metadata, created = (await tasks.backgrounded(
getattr(self.provider, action),
self.dest_provider,
self.path,
self.dest_path,
rename=self.json.get('rename'),
conflict=self.json.get('conflict', DEFAULT_CONFLICT),
))
self.dest_meta = metadata
if created:
self.set_status(201)
else:
self.set_status(200)
self.write({'data': metadata.json_api_serialized(self.dest_resource)})
def move_or_copy(self):
# Force the json body to load into memory
yield self.request.body
if self.json.get('action') not in ('copy', 'move', 'rename'):
# Note: null is used as the default to avoid python specific error messages
raise exceptions.InvalidParameters(
'Action must be copy, move or rename, not {}'.format(
self.json.get('action', 'null')))
if self.json['action'] == 'rename':
action = 'move'
self.dest_auth = self.auth
self.dest_provider = self.provider
self.dest_path = self.path.parent
else:
if 'path' not in self.json:
raise exceptions.InvalidParameters(
'Path is required for moves or copies')
action = self.json['action']
# Note: attached to self so that _send_hook has access to these
self.dest_resource = self.json.get('resource', self.resource)
# TODO optimize for same provider and resource
self.dest_auth = yield from auth_handler.get(
self.dest_resource,
self.json.get('provider', self.provider.NAME), self.request)
self.dest_provider = make_provider(
self.json.get('provider',
self.provider.NAME), self.dest_auth['auth'],
self.dest_auth['credentials'], self.dest_auth['settings'])
self.dest_path = yield from self.dest_provider.validate_path(
self.json['path'])
if not getattr(self.provider, 'can_intra_' + action)(
self.dest_provider, self.path):
result = yield from getattr(tasks, action).adelay(
*self.build_args(self.dest_provider, self.dest_path))
metadata, created = yield from tasks.wait_on_celery(result)
else:
metadata, created = (yield from tasks.backgrounded(
getattr(self.provider, action),
self.dest_provider,
self.path,
self.dest_path,
rename=self.json.get('rename'),
conflict=self.json.get('conflict', 'replace'),
))
metadata = metadata.serialized()
if created:
self.set_status(201)
else:
self.set_status(200)
self.write(metadata)
