def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(
HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message='Invalid URL path.')
request.log_context(agent=request.headers.get('User-Agent'),
path=request_path,
method=request.method,
lang=request.headers.get('Accept-Language'),
errno=0)
qs = dict(errors.request_GET(request))
if qs:
request.log_context(querystring=qs)
if summary_logger.level == logging.DEBUG:
request.log_context(headers=dict(request.headers),
body=request.body)
def fxa_oauth_token(request):
"""Return OAuth token from authorization code.
"""
state = request.validated['querystring']['state']
code = request.validated['querystring']['code']
# Require on-going session
stored_redirect = request.registry.cache.get(state)
# Make sure we cannot try twice with the same code
request.registry.cache.delete(state)
if not stored_redirect:
error_msg = 'The OAuth session was not found, please re-authenticate.'
return http_error(httpexceptions.HTTPRequestTimeout(),
errno=ERRORS.MISSING_AUTH_TOKEN,
message=error_msg)
# Trade the OAuth code for a longer-lived token
auth_client = OAuthClient(server_url=fxa_conf(request, 'oauth_uri'),
client_id=fxa_conf(request, 'client_id'),
client_secret=fxa_conf(request, 'client_secret'))
try:
token = auth_client.trade_code(code)
except fxa_errors.OutOfProtocolError:
raise httpexceptions.HTTPServiceUnavailable()
except fxa_errors.InProtocolError as error:
logger.error(error)
error_details = {
'name': 'code',
'location': 'querystring',
'description': 'Firefox Account code validation failed.'
}
raise_invalid(request, **error_details)
return httpexceptions.HTTPFound(location='%s%s' % (stored_redirect, token))
def _404_for_record(self, record_id):
details = {
'id': record_id,
'resource_name': self.request.current_resource_name
}
return http_error(HTTPNotFound(), errno=ERRORS.INVALID_RESOURCE_ID,
details=details)
def page_not_found(response, request):
"""Return a JSON 404 error response."""
config_key = 'trailing_slash_redirect_enabled'
redirect_enabled = request.registry.settings[config_key]
trailing_slash_redirection_enabled = asbool(redirect_enabled)
querystring = request.url[(request.url.rindex(request.path) +
len(request.path)):]
errno = ERRORS.MISSING_RESOURCE
error_msg = 'The resource you are looking for could not be found.'
if not request.path.startswith('/{}'.format(request.registry.route_prefix)):
errno = ERRORS.VERSION_NOT_AVAILABLE
error_msg = ('The requested API version is not available '
'on this server.')
elif trailing_slash_redirection_enabled:
redirect = None
if request.path.endswith('/'):
path = request.path.rstrip('/')
redirect = '{}{}'.format(path, querystring)
elif request.path == '/{}'.format(request.registry.route_prefix):
# Case for /v0 -> /v0/
redirect = '/{}/{}'.format(request.registry.route_prefix, querystring)
if redirect:
return reapply_cors(request, HTTPTemporaryRedirect(redirect))
if response.content_type != 'application/json':
response = http_error(httpexceptions.HTTPNotFound(),
errno=errno,
message=error_msg)
return reapply_cors(request, response)
def test_403_can_be_overridded(self):
custom_403 = http_error(httpexceptions.HTTPForbidden(), errno=ERRORS.FORBIDDEN,
message="Customized.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_403):
response = self.app.get(self.sample_url, headers=self.headers, status=403)
self.assertFormattedError(response, 403, ERRORS.FORBIDDEN, "Forbidden", "Customized.")
def page_not_found(request):
"""Return a JSON 404 error response."""
config_key = 'trailing_slash_redirect_enabled'
redirect_enabled = request.registry.settings[config_key]
trailing_slash_redirection_enabled = asbool(redirect_enabled)
querystring = request.url[(request.url.rindex(request.path) +
len(request.path)):]
errno = ERRORS.MISSING_RESOURCE
error_msg = "The resource you are looking for could not be found."
if not request.path.startswith('/' + request.registry.route_prefix):
errno = ERRORS.VERSION_NOT_AVAILABLE
error_msg = ("The requested protocol version is not available "
"on this server.")
elif trailing_slash_redirection_enabled:
redirect = None
if request.path.endswith('/'):
path = request.path.rstrip('/')
redirect = '%s%s' % (path, querystring)
elif request.path == '/' + request.registry.route_prefix:
# Case for /v0 -> /v0/
redirect = '/%s/%s' % (request.registry.route_prefix, querystring)
if redirect:
return HTTPTemporaryRedirect(redirect)
response = http_error(httpexceptions.HTTPNotFound(),
errno=errno,
message=error_msg)
return response
def test_405_can_have_custom_message(self):
custom_405 = http_error(httpexceptions.HTTPMethodNotAllowed(),
errno=ERRORS.METHOD_NOT_ALLOWED, message="Disabled from conf.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_405):
response = self.app.get(self.sample_url, headers=self.headers, status=405)
self.assertFormattedError(response, 405, ERRORS.METHOD_NOT_ALLOWED, "Method Not Allowed",
"Disabled from conf.")
def collection_post(self):
result = super(Account, self).collection_post()
if self.context.is_anonymous and self.request.response.status_code == 200:
error_details = {
'message': 'Account ID %r already exists' % result['data']['id']
}
raise http_error(httpexceptions.HTTPForbidden(), **error_details)
return result
def authorization_required(request):
"""Distinguish authentication required (``401 Unauthorized``) from
not allowed (``403 Forbidden``).
"""
if Authenticated not in request.effective_principals:
error_msg = "Please authenticate yourself to use this endpoint."
response = http_error(httpexceptions.HTTPUnauthorized(),
errno=ERRORS.MISSING_AUTH_TOKEN,
message=error_msg)
response.headers.extend(forget(request))
return response
error_msg = "This user cannot access this resource."
response = http_error(httpexceptions.HTTPForbidden(),
errno=ERRORS.FORBIDDEN,
message=error_msg)
return response
def method_not_allowed(context, request):
if context.content_type == 'application/json':
return context
response = http_error(context,
errno=ERRORS.METHOD_NOT_ALLOWED,
message='Method not allowed on this endpoint.')
return reapply_cors(request, response)
def test_503_can_have_custom_message(self):
custom_503 = http_error(httpexceptions.HTTPServiceUnavailable(),
errno=ERRORS.BACKEND,
message="Unable to connect the server.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_503):
response = self.app.get(self.sample_url, headers=self.headers, status=503)
self.assertFormattedError(response, 503, ERRORS.BACKEND, "Service Unavailable",
"Unable to connect the server.")
def object_exists_or_404(request, collection_id, object_id, parent_id=""):
storage = request.registry.storage
try:
return storage.get(collection_id=collection_id, parent_id=parent_id, object_id=object_id)
except exceptions.RecordNotFoundError:
# XXX: We gave up putting details about parent id here (See #53).
details = {"id": object_id, "resource_name": collection_id}
response = http_error(HTTPNotFound(), errno=ERRORS.MISSING_RESOURCE, details=details)
raise response
def test_503_can_have_custom_message(self):
custom_503 = http_error(httpexceptions.HTTPServiceUnavailable(),
errno=ERRORS.BACKEND,
message="Unable to connect the server.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_503):
response = self.app.get(self.sample_url, headers=self.headers, status=503)
self.assertFormattedError(response, 503, ERRORS.BACKEND, "Service Unavailable",
"Unable to connect the server.")
def plural_post(self):
result = super(Account, self).plural_post()
if self.context.is_anonymous and self.request.response.status_code == 200:
error_details = {
"message":
"Account ID %r already exists" % result["data"]["id"]
}
raise http_error(httpexceptions.HTTPForbidden(), **error_details)
return result
def collection_post(self):
result = super(Account, self).collection_post()
if self.context.is_anonymous and self.request.response.status_code == 200:
error_details = {
'message':
'Account ID %r already exists' % result['data']['id']
}
raise http_error(httpexceptions.HTTPForbidden(), **error_details)
return result
def test_403_can_be_overridded(self):
custom_403 = http_error(
httpexceptions.HTTPForbidden(), errno=ERRORS.FORBIDDEN, message="Customized."
)
with mock.patch(
"tests.core.testapp.views.Mushroom._extract_filters", side_effect=custom_403
):
response = self.app.get(self.sample_url, headers=self.headers, status=403)
self.assertFormattedError(response, 403, ERRORS.FORBIDDEN, "Forbidden", "Customized.")
def test_404_can_be_overridded(self):
custom_404 = http_error(httpexceptions.HTTPNotFound(),
errno=ERRORS.MISSING_RESOURCE,
message="Customized.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_404):
response = self.app.get(self.sample_url, headers=self.headers, status=404)
self.assertFormattedError(
response, 404, ERRORS.MISSING_RESOURCE, "Not Found", "Customized.")
def test_404_can_be_overridden(self):
custom_404 = http_error(httpexceptions.HTTPNotFound(),
errno=ERRORS.MISSING_RESOURCE,
message="Customized.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_404):
response = self.app.get(self.sample_url, headers=self.headers, status=404)
self.assertFormattedError(response, 404, ERRORS.MISSING_RESOURCE,
"Not Found", "Customized.")
def authorization_required(response, request):
"""Distinguish authentication required (``401 Unauthorized``) from
not allowed (``403 Forbidden``).
"""
if Authenticated not in request.effective_principals:
error_msg = "Please authenticate yourself to use this endpoint."
response = http_error(httpexceptions.HTTPUnauthorized(),
errno=ERRORS.MISSING_AUTH_TOKEN,
message=error_msg)
response.headers.extend(forget(request))
return response
if response.content_type != "application/json":
error_msg = "This user cannot access this resource."
response = http_error(httpexceptions.HTTPForbidden(),
errno=ERRORS.FORBIDDEN,
message=error_msg)
return reapply_cors(request, response)
def error(context, request):
"""Catch server errors and trace them."""
if isinstance(context, httpexceptions.Response):
return reapply_cors(request, context)
if isinstance(context, storage_exceptions.IntegrityError):
error_msg = 'Integrity constraint violated, please retry.'
response = http_error(httpexceptions.HTTPConflict(),
errno=ERRORS.CONSTRAINT_VIOLATED,
message=error_msg)
retry_after = request.registry.settings['retry_after_seconds']
response.headers['Retry-After'] = str(retry_after)
return reapply_cors(request, response)
# Log some information about current request.
extra = {
'path': request.path,
'method': request.method,
}
qs = dict(request_GET(request))
if qs:
extra['querystring'] = qs
# Take errno from original exception, or undefined if unknown/unhandled.
try:
extra['errno'] = context.errno.value
except AttributeError:
extra['errno'] = ERRORS.UNDEFINED.value
if isinstance(context, storage_exceptions.BackendError):
logger.critical(context.original, extra=extra, exc_info=context)
response = httpexceptions.HTTPServiceUnavailable()
return service_unavailable(response, request)
# Within the exception view, sys.exc_info() will return null.
# see https://github.com/python/cpython/blob/ce9e62544/Lib/logging/__init__.py#L1460-L1462
logger.error(context, extra=extra, exc_info=context)
error_msg = 'A programmatic error occured, developers have been informed.'
info = request.registry.settings['error_info_link']
response = http_error(httpexceptions.HTTPInternalServerError(),
message=error_msg,
info=info)
return reapply_cors(request, response)
def service_unavailable(response, request):
if response.content_type != 'application/json':
error_msg = ('Service temporary unavailable '
'due to overloading or maintenance, please retry later.')
response = http_error(response, errno=ERRORS.BACKEND,
message=error_msg)
retry_after = request.registry.settings['retry_after_seconds']
response.headers['Retry-After'] = str(retry_after)
return reapply_cors(request, response)
def test_405_can_have_custom_message(self):
custom_405 = http_error(httpexceptions.HTTPMethodNotAllowed(),
errno=ERRORS.METHOD_NOT_ALLOWED,
message="Disabled from conf.")
with mock.patch('tests.core.testapp.views.Mushroom._extract_filters',
side_effect=custom_405):
response = self.app.get(self.sample_url,
headers=self.headers,
status=405)
self.assertFormattedError(response, 405, ERRORS.METHOD_NOT_ALLOWED,
"Method Not Allowed", "Disabled from conf.")
def _raise_412_if_modified(self, obj=None):
"""Raise 412 if current timestamp is superior to the one
specified in headers.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed`
"""
if_match = self.request.validated["header"].get("If-Match")
if_none_match = self.request.validated["header"].get("If-None-Match")
# Check if object exists
object_exists = obj is not None
# If no precondition headers, just ignore
if not if_match and not if_none_match:
return
# If-None-Match: * should always raise if an object exists
if if_none_match == "*" and object_exists:
modified_since = -1 # Always raise.
# If-Match should always raise if an object doesn't exist
elif if_match and not object_exists:
modified_since = -1
# If-Match with ETag value on existing objects should compare ETag
elif if_match and if_match != "*":
modified_since = if_match
# If none of the above applies, don't raise
else:
return
if obj:
current_timestamp = obj[self.model.modified_field]
else:
current_timestamp = self.model.timestamp()
if current_timestamp != modified_since:
error_msg = "Resource was modified meanwhile"
# Do not provide the permissions among the object fields.
# Ref: https://github.com/Kinto/kinto/issues/224
existing = {**obj} if obj else {}
existing.pop(self.model.permissions_field, None)
details = {"existing": existing} if obj else {}
response = http_error(
HTTPPreconditionFailed(),
errno=ERRORS.MODIFIED_MEANWHILE,
message=error_msg,
details=details,
)
self._add_timestamp_header(response, timestamp=current_timestamp)
raise response
def _raise_412_if_modified(self, record=None):
"""Raise 412 if current timestamp is superior to the one
specified in headers.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed`
"""
if_match = self.request.headers.get('If-Match')
if_none_match = self.request.headers.get('If-None-Match')
if not if_match and not if_none_match:
return
error_details = {
'location': 'header',
'description': ("Invalid value for If-Match. The value should "
"be integer between double quotes.")}
try:
if_match = decode_header(if_match) if if_match else None
if_none_match = decode_header(if_none_match) if if_none_match else None
except UnicodeDecodeError:
raise_invalid(self.request, **error_details)
if record and if_none_match == '*':
if record.get(self.model.deleted_field, False):
# Tombstones should not prevent creation.
return
modified_since = -1 # Always raise.
elif if_match:
try:
if not (if_match[0] == if_match[-1] == '"'):
raise ValueError()
modified_since = int(if_match[1:-1])
except (IndexError, ValueError):
raise_invalid(self.request, **error_details)
else:
# In case _raise_304_if_not_modified() did not raise.
return
if record:
current_timestamp = record[self.model.modified_field]
else:
current_timestamp = self.model.timestamp()
if current_timestamp > modified_since:
error_msg = 'Resource was modified meanwhile'
details = {'existing': record} if record else {}
response = http_error(HTTPPreconditionFailed(),
errno=ERRORS.MODIFIED_MEANWHILE,
message=error_msg,
details=details)
self._add_timestamp_header(response, timestamp=current_timestamp)
raise response
def _extract_posted_body_id(request):
try:
# Anonymous creation with POST.
return request.json["data"]["id"]
except (ValueError, KeyError):
# Bad POST data.
if request.method.lower() == "post":
error_details = {"name": "data.id", "description": "data.id in body: Required"}
raise_invalid(request, **error_details)
# Anonymous GET
error_msg = "Cannot read accounts."
raise http_error(httpexceptions.HTTPUnauthorized(), error=error_msg)
def object_exists_or_404(request, collection_id, object_id, parent_id=''):
storage = request.registry.storage
try:
return storage.get(collection_id=collection_id,
parent_id=parent_id,
object_id=object_id)
except exceptions.RecordNotFoundError:
# XXX: We gave up putting details about parent id here (See #53).
details = {"id": object_id, "resource_name": collection_id}
response = http_error(HTTPNotFound(),
errno=ERRORS.MISSING_RESOURCE,
details=details)
raise response
def post_validation(request):
user_id = request.matchdict["user_id"]
activation_key = request.matchdict["activation_key"]
parent_id = user_id
try:
user = request.registry.storage.get(parent_id=parent_id,
resource_name="account",
object_id=user_id)
except storage_exceptions.ObjectNotFoundError:
# Don't give information on the existence of a user id: return a generic error message.
error_details = {
"message": "Account ID and activation key do not match"
}
raise http_error(httpexceptions.HTTPForbidden(), **error_details)
if not check_validation_key(activation_key, user_id, request.registry):
error_details = {
"message": "Account ID and activation key do not match"
}
raise http_error(httpexceptions.HTTPForbidden(), **error_details)
# User is now validated.
new = user.copy()
new["validated"] = True
result = request.registry.storage.update(parent_id=parent_id,
resource_name="account",
object_id=user_id,
obj=new)
request.notify_resource_event(
parent_id=parent_id,
timestamp=result["last_modified"],
data=result,
action=ACTIONS.UPDATE,
old=user,
resource_name="account",
)
return new
def search_view(request, **kwargs):
bucket_id = request.matchdict['bucket_id']
collection_id = request.matchdict['collection_id']
# Limit the number of results to return, based on existing Kinto settings.
paginate_by = request.registry.settings.get("paginate_by")
max_fetch_size = request.registry.settings["storage_max_fetch_size"]
if paginate_by is None or paginate_by <= 0:
paginate_by = max_fetch_size
configured = min(paginate_by, max_fetch_size)
# If the size is specified in query, ignore it if larger than setting.
specified = None
if "body" in kwargs:
try:
body = json.loads(kwargs["body"].decode("utf-8"))
specified = body.get("size")
except json.decoder.JSONDecodeError:
pass
if specified is None or specified > configured:
kwargs.setdefault("size", configured)
# Access indexer from views using registry.
indexer = request.registry.indexer
try:
results = indexer.search(bucket_id, collection_id, **kwargs)
except elasticsearch.NotFoundError:
# If plugin was enabled after the creation of the collection.
indexer.create_index(bucket_id, collection_id)
results = indexer.search(bucket_id, collection_id, **kwargs)
except elasticsearch.RequestError as e:
# Malformed query.
if isinstance(e.info["error"], dict):
message = e.info["error"]["reason"]
details = e.info["error"]["root_cause"][0]
else:
message = e.info["error"]
details = None
response = http_error(httpexceptions.HTTPBadRequest(),
errno=ERRORS.INVALID_PARAMETERS,
message=message,
details=details)
raise response
except elasticsearch.ElasticsearchException as e:
# General failure.
logger.exception(f"Index query failed ({e})")
results = {}
return results
def _raise_412_if_modified(self, record=None):
"""Raise 412 if current timestamp is superior to the one
specified in headers.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed`
"""
if_match = self.request.headers.get('If-Match')
if_none_match = self.request.headers.get('If-None-Match')
if not if_match and not if_none_match:
return
if_match = decode_header(if_match) if if_match else None
if record and if_none_match and decode_header(if_none_match) == '*':
if record.get(self.model.deleted_field, False):
# Tombstones should not prevent creation.
return
modified_since = -1 # Always raise.
elif if_match:
try:
if not (if_match[0] == if_match[-1] == '"'):
raise ValueError()
modified_since = int(if_match[1:-1])
except (IndexError, ValueError):
message = ("Invalid value for If-Match. The value should "
"be integer between double quotes.")
error_details = {
'location': 'headers',
'description': message
}
raise_invalid(self.request, **error_details)
else:
# In case _raise_304_if_not_modified() did not raise.
return
if record:
current_timestamp = record[self.model.modified_field]
else:
current_timestamp = self.model.timestamp()
if current_timestamp > modified_since:
error_msg = 'Resource was modified meanwhile'
details = {'existing': record} if record else {}
response = http_error(HTTPPreconditionFailed(),
errno=ERRORS.MODIFIED_MEANWHILE,
message=error_msg,
details=details)
self._add_timestamp_header(response, timestamp=current_timestamp)
raise response
def post_batch(request):
requests = request.validated['requests']
batch_size = len(requests)
limit = request.registry.settings['batch_max_requests']
if limit and len(requests) > int(limit):
error_msg = 'Number of requests is limited to %s' % limit
request.errors.add('body', 'requests', error_msg)
return
if any([batch.path in req['path'] for req in requests]):
error_msg = 'Recursive call on %s endpoint is forbidden.' % batch.path
request.errors.add('body', 'requests', error_msg)
return
responses = []
sublogger = logger.new()
for subrequest_spec in requests:
subrequest = build_request(request, subrequest_spec)
sublogger.bind(path=subrequest.path,
method=subrequest.method)
try:
# Invoke subrequest without individual transaction.
resp, subrequest = request.follow_subrequest(subrequest,
use_tweens=False)
except httpexceptions.HTTPException as e:
if e.content_type == 'application/json':
resp = e
else:
# JSONify raw Pyramid errors.
resp = errors.http_error(e)
sublogger.bind(code=resp.status_code)
sublogger.info('subrequest.summary')
dict_resp = build_response(resp, subrequest)
responses.append(dict_resp)
# Rebing batch request for summary
logger.bind(path=batch.path,
method=request.method,
batch_size=batch_size,
agent=request.headers.get('User-Agent'),)
return {
'responses': responses
}
def post_batch(request):
requests = request.validated['body']['requests']
request.log_context(batch_size=len(requests))
limit = request.registry.settings['batch_max_requests']
if limit and len(requests) > int(limit):
error_msg = 'Number of requests is limited to {}'.format(limit)
request.errors.add('body', 'requests', error_msg)
return
if any([batch.path in req['path'] for req in requests]):
error_msg = 'Recursive call on {} endpoint is forbidden.'.format(batch.path)
request.errors.add('body', 'requests', error_msg)
return
responses = []
for subrequest_spec in requests:
subrequest = build_request(request, subrequest_spec)
log_context = {**request.log_context(),
'path': subrequest.path,
'method': subrequest.method}
try:
# Invoke subrequest without individual transaction.
resp, subrequest = request.follow_subrequest(subrequest,
use_tweens=False)
except httpexceptions.HTTPException as e:
# Since some request in the batch failed, we need to stop the parent request
# through Pyramid's transaction manager. 5XX errors are already caught by
# pyramid_tm's commit_veto
# https://github.com/Kinto/kinto/issues/624
if e.status_code == 409:
request.tm.abort()
if e.content_type == 'application/json':
resp = e
else:
# JSONify raw Pyramid errors.
resp = errors.http_error(e)
subrequest_logger.info('subrequest.summary', extra=log_context)
dict_resp = build_response(resp, subrequest)
responses.append(dict_resp)
return {
'responses': responses
}
def on_update_record(event):
if getattr(event.request, '_attachment_auto_save', False):
# Record attributes are being by the plugin itself.
return
# A user is changing the record, make sure attachment metadata is not
# altered manually.
for change in event.impacted_records:
attachment_before = change['old'].get('attachment')
attachment_after = change['new'].get('attachment')
if attachment_before and attachment_after:
if attachment_before != attachment_after:
error_msg = "Attachment metadata cannot be modified."
raise http_error(HTTPBadRequest(), message=error_msg)
def on_update_record(event):
if getattr(event.request, '_attachment_auto_save', False):
# Record attributes are being by the plugin itself.
return
# A user is changing the record, make sure attachment metadata is not
# altered manually.
for change in event.impacted_records:
attachment_before = change['old'].get('attachment')
attachment_after = change['new'].get('attachment')
if attachment_before and attachment_after:
if attachment_before != attachment_after:
error_msg = "Attachment metadata cannot be modified."
raise http_error(HTTPBadRequest(), message=error_msg)
def post_batch(request):
requests = request.validated["body"]["requests"]
request.log_context(batch_size=len(requests))
limit = request.registry.settings["batch_max_requests"]
if limit and len(requests) > int(limit):
error_msg = f"Number of requests is limited to {limit}"
request.errors.add("body", "requests", error_msg)
return
if any([batch.path in req["path"] for req in requests]):
error_msg = f"Recursive call on {batch.path} endpoint is forbidden."
request.errors.add("body", "requests", error_msg)
return
responses = []
for subrequest_spec in requests:
subrequest = build_request(request, subrequest_spec)
log_context = {
**request.log_context(),
"path": subrequest.path,
"method": subrequest.method,
}
try:
# Invoke subrequest without individual transaction.
resp, subrequest = request.follow_subrequest(subrequest,
use_tweens=False)
except httpexceptions.HTTPException as e:
# Since some request in the batch failed, we need to stop the parent request
# through Pyramid's transaction manager. 5XX errors are already caught by
# pyramid_tm's commit_veto
# https://github.com/Kinto/kinto/issues/624
if e.status_code == 409:
request.tm.abort()
if e.content_type == "application/json":
resp = e
else:
# JSONify raw Pyramid errors.
resp = errors.http_error(e)
subrequest_logger.info("subrequest.summary", extra=log_context)
dict_resp = build_response(resp, subrequest)
responses.append(dict_resp)
return {"responses": responses}
def post_batch(request):
requests = request.validated['body']['requests']
batch_size = len(requests)
limit = request.registry.settings['batch_max_requests']
if limit and len(requests) > int(limit):
error_msg = 'Number of requests is limited to %s' % limit
request.errors.add('body', 'requests', error_msg)
return
if any([batch.path in req['path'] for req in requests]):
error_msg = 'Recursive call on %s endpoint is forbidden.' % batch.path
request.errors.add('body', 'requests', error_msg)
return
responses = []
sublogger = logger.new()
for subrequest_spec in requests:
subrequest = build_request(request, subrequest_spec)
sublogger.bind(path=subrequest.path,
method=subrequest.method)
try:
# Invoke subrequest without individual transaction.
resp, subrequest = request.follow_subrequest(subrequest,
use_tweens=False)
except httpexceptions.HTTPException as e:
if e.content_type == 'application/json':
resp = e
else:
# JSONify raw Pyramid errors.
resp = errors.http_error(e)
sublogger.bind(code=resp.status_code)
sublogger.info('subrequest.summary')
dict_resp = build_response(resp, subrequest)
responses.append(dict_resp)
# Rebing batch request for summary
logger.bind(path=batch.path,
method=request.method,
batch_size=batch_size,
agent=request.headers.get('User-Agent'),)
return {
'responses': responses
}
def _raise_412_if_modified(self, record=None):
"""Raise 412 if current timestamp is superior to the one
specified in headers.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed`
"""
if_match = self.request.validated["header"].get("If-Match")
if_none_match = self.request.validated["header"].get("If-None-Match")
# Check if record exists
record_exists = record is not None
# If no precondition headers, just ignore
if not if_match and not if_none_match:
return
# If-None-Match: * should always raise if a record exists
if if_none_match == "*" and record_exists:
modified_since = -1 # Always raise.
# If-Match should always raise if a record doesn't exist
elif if_match and not record_exists:
modified_since = -1
# If-Match with ETag value on existing records should compare ETag
elif if_match and if_match != "*":
modified_since = if_match
# If none of the above applies, don't raise
else:
return
if record:
current_timestamp = record[self.model.modified_field]
else:
current_timestamp = self.model.timestamp()
if current_timestamp != modified_since:
error_msg = "Resource was modified meanwhile"
details = {"existing": record} if record else {}
response = http_error(
HTTPPreconditionFailed(),
errno=ERRORS.MODIFIED_MEANWHILE,
message=error_msg,
details=details,
)
self._add_timestamp_header(response, timestamp=current_timestamp)
raise response
def test_formatted_error_are_passed_through(self):
response = http_error(HTTPBadRequest(),
errno=ERRORS.INVALID_PARAMETERS,
message='Yop')
with mock.patch.object(self.storage, 'create') as mocked:
mocked.side_effect = [
{"id": "abc", "last_modified": 43},
{"id": "abc", "last_modified": 44},
response
]
resp = self.app.post(self.collection_url + '/records',
headers=self.headers,
status=400)
self.assertEqual(resp.body, response.body)
def _get_record_or_404(self, record_id):
"""Retrieve record from storage and raise ``404 Not found`` if missing.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
"""
if self.context and self.context.current_record:
# Set during authorization. Save a storage hit.
return self.context.current_record
try:
return self.model.get_record(record_id)
except storage_exceptions.RecordNotFoundError:
response = http_error(HTTPNotFound(), errno=ERRORS.INVALID_RESOURCE_ID)
raise response
def _extract_posted_body_id(request):
try:
# Anonymous creation with POST.
return request.json['data']['id']
except (ValueError, KeyError):
# Bad POST data.
if request.method.lower() == 'post':
error_details = {
'name': 'data.id',
'description': 'data.id in body: Required'
}
raise_invalid(request, **error_details)
# Anonymous GET
error_msg = 'Cannot read accounts.'
raise http_error(httpexceptions.HTTPUnauthorized(), error=error_msg)
def _extract_posted_body_id(request):
try:
# Anonymous creation with POST.
return request.json['data']['id']
except (ValueError, KeyError):
# Bad POST data.
if request.method.lower() == 'post':
error_details = {
'name': 'data.id',
'description': 'data.id in body: Required'
}
raise_invalid(request, **error_details)
# Anonymous GET
error_msg = 'Cannot read accounts.'
raise http_error(httpexceptions.HTTPUnauthorized(), error=error_msg)
def facebook_token(request):
"""Return OAuth token from authorization code.
"""
state = request.validated['querystring']['state']
code = request.validated['querystring']['code']
# Require on-going session
stored_redirect = request.registry.cache.get(state)
# Make sure we cannot try twice with the same code
request.registry.cache.delete(state)
if not stored_redirect:
error_msg = 'The Facebook Auth session was not found, please re-authenticate.'
return http_error(httpexceptions.HTTPRequestTimeout(),
errno=ERRORS.MISSING_AUTH_TOKEN,
message=error_msg)
url = facebook_conf(request, 'token_endpoint')
params = {
'client_id': facebook_conf(request, 'client_id'),
'client_secret': facebook_conf(request, 'client_secret'),
'redirect_uri': request.route_url(token.name),
'code': code,
}
resp = requests.get(url, params=params)
if resp.status_code == 400:
response_body = resp.json()
logger.error(
"Facebook Token Validation Failed: {}".format(response_body))
error_details = {
'name': 'code',
'location': 'querystring',
'description': 'Facebook OAuth code validation failed.'
}
raise_invalid(request, **error_details)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError:
logger.exception("Facebook Token Protocol Error")
raise httpexceptions.HTTPServiceUnavailable()
else:
response_body = resp.json()
access_token = response_body['access_token']
return httpexceptions.HTTPFound(location='%s%s' %
(stored_redirect, access_token))
def test_formatted_error_are_passed_through(self):
# Create the parent objects
self.app.post(self.collection_url + "/records", headers=self.headers)
# Simulate a validation error.
fake400 = http_error(HTTPBadRequest(), errno=ERRORS.INVALID_PARAMETERS, message="Yop")
with mock.patch.object(self.storage, "create") as mocked:
mocked.side_effect = [
{"id": "abc", "last_modified": 43},
{"id": "abc", "last_modified": 44},
fake400,
]
resp = self.app.post(
self.collection_url + "/records", headers=self.headers, status=400
)
self.assertEqual(resp.body, fake400.body)
def _get_record_or_404(self, record_id):
"""Retrieve record from storage and raise ``404 Not found`` if missing.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
"""
if self.context and self.context.current_record:
# Set during authorization. Save a storage hit.
return self.context.current_record
try:
return self.model.get_record(record_id)
except storage_exceptions.RecordNotFoundError:
response = http_error(HTTPNotFound(),
errno=ERRORS.INVALID_RESOURCE_ID)
raise response
def _raise_412_if_modified(self, record=None):
"""Raise 412 if current timestamp is superior to the one
specified in headers.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed`
"""
if_match = self.request.validated['header'].get('If-Match')
if_none_match = self.request.validated['header'].get('If-None-Match')
# Check if record exists
record_exists = record is not None
# If no precondition headers, just ignore
if not if_match and not if_none_match:
return
# If-None-Match: * should always raise if a record exists
if if_none_match == '*' and record_exists:
modified_since = -1 # Always raise.
# If-Match should always raise if a record doesn't exist
elif if_match and not record_exists:
modified_since = -1
# If-Match with ETag value on existing records should compare ETag
elif if_match and if_match != '*':
modified_since = if_match
# If none of the above applies, don't raise
else:
return
if record:
current_timestamp = record[self.model.modified_field]
else:
current_timestamp = self.model.timestamp()
if current_timestamp != modified_since:
error_msg = 'Resource was modified meanwhile'
details = {'existing': record} if record else {}
response = http_error(HTTPPreconditionFailed(),
errno=ERRORS.MODIFIED_MEANWHILE,
message=error_msg,
details=details)
self._add_timestamp_header(response, timestamp=current_timestamp)
raise response
def post_manual_step(request):
# Get the record.
answer = request.validated['answer']
record = request.validated['record']
try:
# Set up an AWS stepfunction client.
access_key, secret_key = request.registry.aws_credentials
client = boto3.client(
'stepfunctions',
region_name='us-west-2',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
# Make sure the activity we have in the record is still pending.
activity_arn = get_activity_arn(record, client)
print("activityArn:", activity_arn)
# And get its task token if we didn't already have it.
task_token = record.get('taskToken')
if task_token is None:
task_token = get_task_token(activity_arn, client)
print("Got task token:", task_token)
record['taskToken'] = task_token
update_record(request, record)
# Post a succeed or fail to the stepfunction's activity.
if answer == "FAIL":
print("Sending a FAIL to the aws stepfunction")
client.send_task_failure(
taskToken=task_token,
error="Not signing off",
cause="After reviewing, the decision was made to not sign off")
record['status'] = "FAIL"
else:
print("Sending a SUCCEED to the aws stepfunction")
client.send_task_success(
taskToken=task_token,
output='{"message": "signed off"}')
record['status'] = "SUCCEED"
update_record(request, record)
except Exception as err:
raise http_error(httpexceptions.HTTPServiceUnavailable,
errno=ERRORS.BACKEND,
message=err)
def post_batch(request):
requests = request.validated['body']['requests']
request.log_context(batch_size=len(requests))
limit = request.registry.settings['batch_max_requests']
if limit and len(requests) > int(limit):
error_msg = 'Number of requests is limited to {}'.format(limit)
request.errors.add('body', 'requests', error_msg)
return
if any([batch.path in req['path'] for req in requests]):
error_msg = 'Recursive call on {} endpoint is forbidden.'.format(
batch.path)
request.errors.add('body', 'requests', error_msg)
return
responses = []
for subrequest_spec in requests:
subrequest = build_request(request, subrequest_spec)
log_context = {
'path': subrequest.path,
'method': subrequest.method,
**request.log_context()
}
try:
# Invoke subrequest without individual transaction.
resp, subrequest = request.follow_subrequest(subrequest,
use_tweens=False)
except httpexceptions.HTTPException as e:
if e.content_type == 'application/json':
resp = e
else:
# JSONify raw Pyramid errors.
resp = errors.http_error(e)
subrequest_logger.info('subrequest.summary', extra=log_context)
dict_resp = build_response(resp, subrequest)
responses.append(dict_resp)
return {'responses': responses}
def _raise_conflict(self, exception):
"""Helper to raise conflict responses.
:param exception: the original unicity error
:type exception: :class:`kinto.core.storage.exceptions.UnicityError`
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPConflict`
"""
field = exception.field
record_id = exception.record[self.model.id_field]
message = 'Conflict of field %s on record %s' % (field, record_id)
details = {
"field": field,
"existing": exception.record,
}
response = http_error(HTTPConflict(),
errno=ERRORS.CONSTRAINT_VIOLATED,
message=message,
details=details)
raise response
def _raise_conflict(self, exception):
"""Helper to raise conflict responses.
:param exception: the original unicity error
:type exception: :class:`kinto.core.storage.exceptions.UnicityError`
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPConflict`
"""
field = exception.field
record_id = exception.record[self.model.id_field]
message = 'Conflict of field %s on record %s' % (field, record_id)
details = {
"field": field,
"existing": exception.record,
}
response = http_error(HTTPConflict(),
errno=ERRORS.CONSTRAINT_VIOLATED,
message=message,
details=details)
raise response
def error(context, request):
"""Catch server errors and trace them."""
if isinstance(context, httpexceptions.Response):
return reapply_cors(request, context)
if isinstance(context, storage_exceptions.BackendError):
logger.critical(context.original, exc_info=True)
response = httpexceptions.HTTPServiceUnavailable()
return service_unavailable(response, request)
logger.error(context, exc_info=True)
error_msg = "A programmatic error occured, developers have been informed."
info = request.registry.settings['error_info_link']
response = http_error(httpexceptions.HTTPInternalServerError(),
message=error_msg,
info=info)
return reapply_cors(request, response)
def eos_tween(request):
eos_date = registry.settings["eos"]
eos_url = registry.settings["eos_url"]
eos_message = registry.settings["eos_message"]
if not eos_date:
return handler(request)
eos_date = dateparser.parse(eos_date)
if eos_date > datetime.now():
code = "soft-eol"
request.response = handler(request)
else:
code = "hard-eol"
request.response = errors.http_error(
HTTPGone(), errno=errors.ERRORS.SERVICE_DEPRECATED, message=deprecation_msg
)
errors.send_alert(request, eos_message, url=eos_url, code=code)
return request.response
def error(context, request):
"""Catch server errors and trace them."""
if isinstance(context, httpexceptions.Response):
return reapply_cors(request, context)
if isinstance(context, storage_exceptions.BackendError):
logger.critical(context.original, exc_info=True)
response = httpexceptions.HTTPServiceUnavailable()
return service_unavailable(response, request)
logger.error(context, exc_info=True)
error_msg = "A programmatic error occured, developers have been informed."
info = request.registry.settings['error_info_link']
response = http_error(httpexceptions.HTTPInternalServerError(),
message=error_msg,
info=info)
return reapply_cors(request, response)
def eos_tween(request):
eos_date = registry.settings['eos']
eos_url = registry.settings['eos_url']
eos_message = registry.settings['eos_message']
if not eos_date:
return handler(request)
eos_date = dateparser.parse(eos_date)
if eos_date > datetime.now():
code = "soft-eol"
request.response = handler(request)
else:
code = "hard-eol"
request.response = errors.http_error(
HTTPGone(),
errno=errors.ERRORS.SERVICE_DEPRECATED,
message=deprecation_msg)
errors.send_alert(request, eos_message, url=eos_url, code=code)
return request.response
def portier_verify(request):
"""Helper to redirect client towards Portier login form."""
broker_uri = portier_conf(request, 'broker_uri')
token = request.validated['body']['id_token']
# Get the data from the config because the request might only
# have local network information and not the public facing ones.
audience = '{scheme}://{host}'.format(
scheme=request.registry.settings['http_scheme'],
host=request.registry.settings['http_host'])
try:
email, stored_redirect = get_verified_email(
broker_url=broker_uri,
token=token,
audience=audience,
issuer=broker_uri,
cache=request.registry.cache)
except ValueError as exc:
error_details = 'Portier token validation failed: %s' % exc
return http_error(httpexceptions.HTTPBadRequest(),
errno=ERRORS.INVALID_AUTH_TOKEN,
error='Invalid Auth Token',
message=error_details)
# Generate a random token
user_token = codecs.encode(os.urandom(32), 'hex').decode('utf-8')
# Encrypt the email with the token
encrypted_email = encrypt(email, user_token)
# Generate a user ID from the token
hmac_secret = request.registry.settings['userid_hmac_secret']
userID = utils.hmac_digest(hmac_secret, user_token)
# Store the encrypted user ID with the token
session_ttl = portier_conf(request, 'session_ttl_seconds')
request.registry.cache.set('portier:' + userID, encrypted_email,
session_ttl)
location = '%s%s' % (stored_redirect, user_token)
return httpexceptions.HTTPFound(location=location)
def timestamp(self):
"""Return the current collection timestamp.
:rtype: int
"""
try:
return self.model.timestamp()
except storage_exceptions.BackendError as e:
is_readonly = self.request.registry.settings['readonly']
if not is_readonly:
raise e
# If the instance is configured to be readonly, and if the
# collection is empty, the backend will try to bump the timestamp.
# It fails if the configured db user has not write privileges.
logger.exception(e)
error_msg = ("Collection timestamp cannot be written. "
"Records endpoint must be hit at least once from a "
"writable instance.")
raise http_error(HTTPServiceUnavailable(),
errno=ERRORS.BACKEND,
message=error_msg)
def timestamp(self):
"""Return the current collection timestamp.
:rtype: int
"""
try:
return self.model.timestamp()
except storage_exceptions.BackendError as e:
is_readonly = self.request.registry.settings['readonly']
if not is_readonly:
raise e
# If the instance is configured to be readonly, and if the
# collection is empty, the backend will try to bump the timestamp.
# It fails if the configured db user has not write privileges.
logger.exception(e)
error_msg = ("Collection timestamp cannot be written. "
"Records endpoint must be hit at least once from a "
"writable instance.")
raise http_error(HTTPServiceUnavailable(),
errno=ERRORS.BACKEND,
message=error_msg)
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(
HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message="Invalid URL path.")
# New logger context, with infos for request summary logger.
logger.new(agent=request.headers.get('User-Agent'),
path=request_path,
method=request.method,
querystring=dict(request.GET),
lang=request.headers.get('Accept-Language'),
uid=None,
authn_type=None,
errno=None)
