def on_delete(self, req, resp, project_id, queue_name, message_id):
LOG.debug(u'Messages item DELETE - message: %(message)s, '
u'queue: %(queue)s, project: %(project)s',
{'message': message_id,
'queue': queue_name,
'project': project_id})
try:
self.message_controller.delete(
queue_name,
message_id=message_id,
project=project_id,
claim=req.get_param('claim_id'))
except storage_errors.NotPermitted as ex:
LOG.exception(ex)
title = _(u'Unable to delete')
description = _(u'This message is claimed; it cannot be '
u'deleted without a valid claim_id.')
raise falcon.HTTPForbidden(title, description)
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be deleted.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Alles guete
resp.status = falcon.HTTP_204
def get_checked_field(document, name, value_type):
"""Validates and retrieves a typed field from a document.
This function attempts to look up doc[name], and raises
appropriate HTTP errors if the field is missing or not an
instance of the given type.
:param document: dict-like object
:param name: field name
:param value_type: expected value type, or '*' to accept any type
:raises: HTTPBadRequest if the field is missing or not an
instance of value_type
:returns: value obtained from doc[name]
"""
try:
value = document[name]
except KeyError:
description = _(u'Missing "{name}" field.').format(name=name)
raise errors.HTTPBadRequestBody(description)
if value_type == '*' or isinstance(value, value_type):
return value
description = _(u'The value of the "{name}" field must be a {vtype}.')
description = description.format(name=name, vtype=value_type.__name__)
raise errors.HTTPBadRequestBody(description)
def claim_creation(self, metadata, limit=None):
"""Restrictions on the claim parameters upon creation.
:param metadata: The claim metadata
:param limit: The number of messages to claim
:raises: ValidationFailed if either TTL or grace is out of range,
or the expected number of messages exceed the limit.
"""
self.claim_updating(metadata)
uplimit = self._limits_conf.max_messages_per_claim_or_pop
if limit is not None and not (0 < limit <= uplimit):
msg = _(u'Limit must be at least 1 and may not '
'be greater than {0}.')
raise ValidationFailed(
msg, self._limits_conf.max_messages_per_claim_or_pop)
grace = metadata['grace']
if not (MIN_CLAIM_GRACE <= grace <= self._limits_conf.max_claim_grace):
msg = _(u'The grace for a claim may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(
msg, self._limits_conf.max_claim_grace, MIN_CLAIM_GRACE)
def get_checked_field(document, name, value_type):
"""Validates and retrieves a typed field from a document.
This function attempts to look up doc[name], and raises
appropriate HTTP errors if the field is missing or not an
instance of the given type.
:param document: dict-like object
:param name: field name
:param value_type: expected value type, or '*' to accept any type
:raises: HTTPBadRequest if the field is missing or not an
instance of value_type
:returns: value obtained from doc[name]
"""
try:
value = document[name]
except KeyError:
description = _(u'Missing "{name}" field.').format(name=name)
raise errors.HTTPBadRequestBody(description)
if value_type == '*' or isinstance(value, value_type):
return value
description = _(u'The value of the "{name}" field must be a {vtype}.')
description = description.format(name=name, vtype=value_type.__name__)
raise errors.HTTPBadRequestBody(description)
def on_delete(self, req, resp, project_id, queue_name, message_id):
LOG.debug(
u'Messages item DELETE - message: %(message)s, '
u'queue: %(queue)s, project: %(project)s', {
'message': message_id,
'queue': queue_name,
'project': project_id
})
try:
self.message_controller.delete(queue_name,
message_id=message_id,
project=project_id,
claim=req.get_param('claim_id'))
except storage_errors.NotPermitted as ex:
LOG.exception(ex)
title = _(u'Unable to delete')
description = _(u'This message is claimed; it cannot be '
u'deleted without a valid claim_id.')
raise falcon.HTTPForbidden(title, description)
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be deleted.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Alles guete
resp.status = falcon.HTTP_204
def claim_creation(self, metadata, limit=None):
"""Restrictions on the claim parameters upon creation.
:param metadata: The claim metadata
:param limit: The number of messages to claim
:raises: ValidationFailed if either TTL or grace is out of range,
or the expected number of messages exceed the limit.
"""
self.claim_updating(metadata)
uplimit = self._limits_conf.max_messages_per_claim_or_pop
if limit is not None and not (0 < limit <= uplimit):
msg = _(u'Limit must be at least 1 and may not '
'be greater than {0}.')
raise ValidationFailed(
msg, self._limits_conf.max_messages_per_claim_or_pop)
grace = metadata['grace']
if not (MIN_CLAIM_GRACE <= grace <= self._limits_conf.max_claim_grace):
msg = _(u'The grace for a claim may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(msg, self._limits_conf.max_claim_grace,
MIN_CLAIM_GRACE)
class HTTPServiceUnavailable(falcon.HTTPServiceUnavailable):
"""Wraps falcon.HTTPServiceUnavailable with Marconi messaging."""
TITLE = _(u'Service temporarily unavailable')
DESCRIPTION = _(u'Please try again in a few seconds.')
def __init__(self, description, retry_after=30):
description = description + ' ' + self.DESCRIPTION
super(HTTPServiceUnavailable, self).__init__(self.TITLE, description,
retry_after)
def on_get(self, req, resp, project_id, queue_name, message_id):
LOG.debug(u'Messages item GET - message: %(message)s, '
u'queue: %(queue)s, project: %(project)s',
{'message': message_id,
'queue': queue_name,
'project': project_id})
try:
message = self.message_controller.get(
queue_name,
message_id,
project=project_id)
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be retrieved.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Prepare response
message['href'] = req.path
del message['id']
resp.content_location = req.relative_uri
resp.body = utils.to_json(message)
class HTTPDocumentTypeNotSupported(HTTPBadRequestBody):
"""Wraps HTTPBadRequestBody with a standard description."""
DESCRIPTION = _(u'Document type not supported.')
def __init__(self):
super(HTTPDocumentTypeNotSupported, self).__init__(self.DESCRIPTION)
class HTTPBadRequestBody(falcon.HTTPBadRequest):
"""Wraps falcon.HTTPBadRequest with a contextual title."""
TITLE = _(u'Invalid request body')
def __init__(self, description):
super(HTTPBadRequestBody, self).__init__(self.TITLE, description)
def on_patch(self, req, resp, project_id, queue_name, claim_id):
LOG.debug(u'Claim Item PATCH - claim: %(claim_id)s, '
u'queue: %(queue_name)s, project:%(project_id)s' %
{'queue_name': queue_name,
'project_id': project_id,
'claim_id': claim_id})
# Read claim metadata (e.g., TTL) and raise appropriate
# HTTP errors as needed.
metadata, = wsgi_utils.filter_stream(req.stream, req.content_length,
CLAIM_PATCH_SPEC)
try:
self._validate.claim_updating(metadata)
self.claim_controller.update(queue_name,
claim_id=claim_id,
metadata=metadata,
project=project_id)
resp.status = falcon.HTTP_204
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Claim could not be updated.')
raise wsgi_errors.HTTPServiceUnavailable(description)
def _get_by_id(self, base_path, project_id, queue_name, ids):
"""Returns one or more messages from the queue by ID."""
try:
self._validate.message_listing(limit=len(ids))
messages = self.message_controller.bulk_get(
queue_name,
message_ids=ids,
project=project_id)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be retrieved.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Prepare response
messages = list(messages)
if not messages:
return None
base_path += '/'
for each_message in messages:
each_message['href'] = base_path + each_message['id']
del each_message['id']
return messages
def _get_by_id(self, base_path, project_id, queue_name, ids):
"""Returns one or more messages from the queue by ID."""
try:
self._validate.message_listing(limit=len(ids))
messages = self.message_controller.bulk_get(queue_name,
message_ids=ids,
project=project_id)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be retrieved.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Prepare response
messages = list(messages)
if not messages:
return None
base_path += '/'
for each_message in messages:
each_message['href'] = base_path + each_message['id']
del each_message['id']
return messages
def on_get(self, req, resp, project_id, queue_name, message_id):
LOG.debug(
u'Messages item GET - message: %(message)s, '
u'queue: %(queue)s, project: %(project)s', {
'message': message_id,
'queue': queue_name,
'project': project_id
})
try:
message = self.message_controller.get(queue_name,
message_id,
project=project_id)
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Message could not be retrieved.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Prepare response
message['href'] = req.path
del message['id']
resp.content_location = req.relative_uri
resp.body = utils.to_json(message)
def _pop_messages(self, queue_name, project_id, pop_limit):
try:
LOG.debug(
u'POP messages - queue: %(queue)s, '
u'project: %(project)s', {
'queue': queue_name,
'project': project_id
})
messages = self.message_controller.pop(queue_name,
project=project_id,
limit=pop_limit)
except Exception as ex:
LOG.exception(ex)
description = _(u'Messages could not be popped.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Prepare response
if not messages:
messages = []
body = {'messages': messages}
body = utils.to_json(body)
return falcon.HTTP_200, body
def on_get(self, req, resp, project_id, queue_name):
try:
resp_dict = self.queue_ctrl.stats(queue_name,
project=project_id)
message_stats = resp_dict['messages']
if message_stats['total'] != 0:
base_path = req.path[:req.path.rindex('/')] + '/messages/'
newest = message_stats['newest']
newest['href'] = base_path + newest['id']
del newest['id']
oldest = message_stats['oldest']
oldest['href'] = base_path + oldest['id']
del oldest['id']
resp.content_location = req.path
resp.body = utils.to_json(resp_dict)
# status defaults to 200
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Queue stats could not be read.')
raise wsgi_errors.HTTPServiceUnavailable(description)
def _wrapper():
try:
logging.setup('marconi')
func()
except KeyboardInterrupt:
LOG.info(_(u'Terminating'))
except Exception as ex:
_fail(1, ex)
def _wrapper():
try:
logging.setup("marconi")
func()
except KeyboardInterrupt:
LOG.info(_(u"Terminating"))
except Exception as ex:
_fail(1, ex)
def _inc_counter(self, name, project=None, amount=1, window=None):
"""Increments the message counter and returns the new value.
:param name: Name of the queue to which the counter is scoped
:param project: Queue's project name
:param amount: (Default 1) Amount by which to increment the counter
:param window: (Default None) A time window, in seconds, that
must have elapsed since the counter was last updated, in
order to increment the counter.
:returns: Updated message counter value, or None if window
was specified, and the counter has already been updated
within the specified time period.
:raises: storage.errors.QueueDoesNotExist
"""
now = timeutils.utcnow_ts()
update = {'$inc': {'c.v': amount}, '$set': {'c.t': now}}
query = _get_scoped_query(name, project)
if window is not None:
threshold = now - window
query['c.t'] = {'$lt': threshold}
while True:
try:
doc = self._collection.find_and_modify(query,
update,
new=True,
fields={
'c.v': 1,
'_id': 0
})
break
except pymongo.errors.AutoReconnect as ex:
LOG.exception(ex)
if doc is None:
if window is None:
# NOTE(kgriffs): Since we did not filter by a time window,
# the queue should have been found and updated. Perhaps
# the queue has been deleted?
message = _(u'Failed to increment the message '
u'counter for queue %(name)s and '
u'project %(project)s')
message %= dict(name=name, project=project)
LOG.warning(message)
raise errors.QueueDoesNotExist(name, project)
# NOTE(kgriffs): Assume the queue existed, but the counter
# was recently updated, causing the range query on 'c.t' to
# exclude the record.
return None
return doc['c']['v']
def consumer(*args, **kwargs):
"""Consumes the pipeline for `method`
This function walks through the pipeline and calls
`method` for each of the items in the pipeline. A
warning will be logged for each stage not implementing
`method` and an Attribute error will be raised if
none of the stages do.
:param args: Positional arguments to pass to the call.
:param kwargs: Keyword arguments to pass to the call.
:raises: AttributeError if none of the stages implement `method`
"""
# NOTE(flaper87): Used as a way to verify
# the requested method exists in at least
# one of the stages, otherwise AttributeError
# will be raised.
target = None
for stage in self._pipeline:
try:
target = getattr(stage, method)
except AttributeError:
sstage = six.text_type(stage)
msgtmpl = _(u"Stage %(stage)s does not "
"implement %(method)s")
LOG.warning(msgtmpl, {'stage': sstage, 'method': method})
continue
result = target(*args, **kwargs)
# NOTE(flaper87): Will keep going forward
# through the stageline unless the call returns
# something.
if result is not None:
return result
if target is None:
msg = _(u'Method %s not found in any of '
'the registered stages') % method
LOG.error(msg)
raise AttributeError(msg)
def consumer(*args, **kwargs):
"""Consumes the pipeline for `method`
This function walks through the pipeline and calls
`method` for each of the items in the pipeline. A
warning will be logged for each stage not implementing
`method` and an Attribute error will be raised if
none of the stages do.
:param args: Positional arguments to pass to the call.
:param kwargs: Keyword arguments to pass to the call.
:raises: AttributeError if none of the stages implement `method`
"""
# NOTE(flaper87): Used as a way to verify
# the requested method exists in at least
# one of the stages, otherwise AttributeError
# will be raised.
target = None
for stage in self._pipeline:
try:
target = getattr(stage, method)
except AttributeError:
sstage = six.text_type(stage)
msgtmpl = _(u"Stage %(stage)s does not "
"implement %(method)s")
LOG.warning(msgtmpl, {'stage': sstage, 'method': method})
continue
result = target(*args, **kwargs)
# NOTE(flaper87): Will keep going forward
# through the stageline unless the call returns
# something.
if result is not None:
return result
if target is None:
msg = _(u'Method %s not found in any of '
'the registered stages') % method
LOG.error(msg)
raise AttributeError(msg)
def _delete_messages_by_id(self, queue_name, ids, project_id):
try:
self.message_controller.bulk_delete(queue_name, message_ids=ids, project=project_id)
except Exception as ex:
LOG.exception(ex)
description = _(u"Messages could not be deleted.")
raise wsgi_errors.HTTPServiceUnavailable(description)
return falcon.HTTP_204
def _get(self, req, project_id, queue_name):
client_uuid = wsgi_utils.get_client_uuid(req)
kwargs = {}
# NOTE(kgriffs): This syntax ensures that
# we don't clobber default values with None.
req.get_param('marker', store=kwargs)
req.get_param_as_int('limit', store=kwargs)
req.get_param_as_bool('echo', store=kwargs)
req.get_param_as_bool('include_claimed', store=kwargs)
try:
self._validate.message_listing(**kwargs)
results = self.message_controller.list(
queue_name,
project=project_id,
client_uuid=client_uuid,
**kwargs)
# Buffer messages
cursor = next(results)
messages = list(cursor)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Messages could not be listed.')
raise wsgi_errors.HTTPServiceUnavailable(description)
if not messages:
messages = []
else:
# Found some messages, so prepare the response
kwargs['marker'] = next(results)
for each_message in messages:
each_message['href'] = req.path + '/' + each_message['id']
del each_message['id']
return {
'messages': messages,
'links': [
{
'rel': 'next',
'href': req.path + falcon.to_query_str(kwargs)
}
]
}
def validate_queue_identification(validate, req, resp, params):
"""Hook for validating the queue name and project id in requests.
The queue name validation is short-circuited if 'queue_name' does
not exist in `params`.
This hook depends on the `get_project` hook, which must be
installed upstream.
:param validate: A validator function that will
be used to check the queue name against configured
limits. functools.partial or a closure must be used to
set this first arg, and expose the remaining ones as
a Falcon hook interface.
:param req: Falcon request object
:param resp: Falcon response object
:param params: Responder params dict
"""
try:
validate(params['queue_name'], params['project_id'])
except KeyError:
# NOTE(kgriffs): queue_name not in params, so nothing to do
pass
except validation.ValidationFailed:
project = params['project_id']
queue = params['queue_name']
if six.PY2:
queue = queue.decode('utf-8', 'replace')
LOG.debug(
u'Invalid queue name "%(queue)s" submitted for '
u'project: %(project)s', {
'queue': queue,
'project': project
})
raise falcon.HTTPBadRequest(
_(u'Invalid queue identification'),
_(u'The format of the submitted queue '
u'name or project id is not valid.'))
def queue_metadata_length(self, content_length):
"""Restrictions on queue's length.
:param content_length: Queue request's length.
:raises: ValidationFailed if the metadata is oversize.
"""
if content_length is None:
return
if content_length > self._limits_conf.max_queue_metadata:
msg = _(u'Queue metadata is too large. Max size: {0}')
raise ValidationFailed(msg, self._limits_conf.max_queue_metadata)
def message_content(self, message):
"""Restrictions on each message."""
ttl = message['ttl']
if not (MIN_MESSAGE_TTL <= ttl <= self._limits_conf.max_message_ttl):
msg = _(u'The TTL for a message may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(msg, self._limits_conf.max_message_ttl,
MIN_MESSAGE_TTL)
def queue_metadata_length(self, content_length):
"""Restrictions on queue's length.
:param content_length: Queue request's length.
:raises: ValidationFailed if the metadata is oversize.
"""
if content_length is None:
return
if content_length > self._limits_conf.max_queue_metadata:
msg = _(u'Queue metadata is too large. Max size: {0}')
raise ValidationFailed(msg, self._limits_conf.max_queue_metadata)
def listen(self):
"""Self-host using 'bind' and 'port' from the WSGI config group."""
msgtmpl = _(u'Serving on host %(bind)s:%(port)s')
LOG.info(msgtmpl,
{'bind': self._wsgi_conf.bind, 'port': self._wsgi_conf.port})
httpd = simple_server.make_server(self._wsgi_conf.bind,
self._wsgi_conf.port,
self.app)
httpd.serve_forever()
def message_content(self, message):
"""Restrictions on each message."""
ttl = message['ttl']
if not (MIN_MESSAGE_TTL <= ttl <= self._limits_conf.max_message_ttl):
msg = _(u'The TTL for a message may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(
msg, self._limits_conf.max_message_ttl, MIN_MESSAGE_TTL)
def _inc_counter(self, name, project=None, amount=1, window=None):
"""Increments the message counter and returns the new value.
:param name: Name of the queue to which the counter is scoped
:param project: Queue's project name
:param amount: (Default 1) Amount by which to increment the counter
:param window: (Default None) A time window, in seconds, that
must have elapsed since the counter was last updated, in
order to increment the counter.
:returns: Updated message counter value, or None if window
was specified, and the counter has already been updated
within the specified time period.
:raises: storage.errors.QueueDoesNotExist
"""
now = timeutils.utcnow_ts()
update = {'$inc': {'c.v': amount}, '$set': {'c.t': now}}
query = _get_scoped_query(name, project)
if window is not None:
threshold = now - window
query['c.t'] = {'$lt': threshold}
while True:
try:
doc = self._collection.find_and_modify(
query, update, new=True, fields={'c.v': 1, '_id': 0})
break
except pymongo.errors.AutoReconnect as ex:
LOG.exception(ex)
if doc is None:
if window is None:
# NOTE(kgriffs): Since we did not filter by a time window,
# the queue should have been found and updated. Perhaps
# the queue has been deleted?
message = _(u'Failed to increment the message '
u'counter for queue %(name)s and '
u'project %(project)s')
message %= dict(name=name, project=project)
LOG.warning(message)
raise errors.QueueDoesNotExist(name, project)
# NOTE(kgriffs): Assume the queue existed, but the counter
# was recently updated, causing the range query on 'c.t' to
# exclude the record.
return None
return doc['c']['v']
def message_length(self, content_length):
"""Restrictions on message post length.
:param content_length: Queue request's length.
:raises: ValidationFailed if the metadata is oversize.
"""
if content_length is None:
return
if content_length > self._limits_conf.max_message_size:
raise ValidationFailed(
_(u'Message collection size is too large. Max size {0}'),
self._limits_conf.max_message_size)
def queue_listing(self, limit=None, **kwargs):
"""Restrictions involving a list of queues.
:param limit: The expected number of queues in the list
:param kwargs: Ignored arguments passed to storage API
:raises: ValidationFailed if the limit is exceeded
"""
uplimit = self._limits_conf.max_queues_per_page
if limit is not None and not (0 < limit <= uplimit):
msg = _(u'Limit must be at least 1 and no greater than {0}.')
raise ValidationFailed(msg, self._limits_conf.max_queues_per_page)
def _get(self, req, project_id, queue_name):
client_uuid = wsgi_utils.get_client_uuid(req)
kwargs = {}
# NOTE(kgriffs): This syntax ensures that
# we don't clobber default values with None.
req.get_param('marker', store=kwargs)
req.get_param_as_int('limit', store=kwargs)
req.get_param_as_bool('echo', store=kwargs)
req.get_param_as_bool('include_claimed', store=kwargs)
try:
self._validate.message_listing(**kwargs)
results = self.message_controller.list(queue_name,
project=project_id,
client_uuid=client_uuid,
**kwargs)
# Buffer messages
cursor = next(results)
messages = list(cursor)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except storage_errors.DoesNotExist as ex:
LOG.debug(ex)
raise falcon.HTTPNotFound()
except Exception as ex:
LOG.exception(ex)
description = _(u'Messages could not be listed.')
raise wsgi_errors.HTTPServiceUnavailable(description)
if not messages:
messages = []
else:
# Found some messages, so prepare the response
kwargs['marker'] = next(results)
for each_message in messages:
each_message['href'] = req.path + '/' + each_message['id']
del each_message['id']
return {
'messages':
messages,
'links': [{
'rel': 'next',
'href': req.path + falcon.to_query_str(kwargs)
}]
}
def _delete_messages_by_id(self, queue_name, ids, project_id):
try:
self.message_controller.bulk_delete(queue_name,
message_ids=ids,
project=project_id)
except Exception as ex:
LOG.exception(ex)
description = _(u'Messages could not be deleted.')
raise wsgi_errors.HTTPServiceUnavailable(description)
return falcon.HTTP_204
def queue_listing(self, limit=None, **kwargs):
"""Restrictions involving a list of queues.
:param limit: The expected number of queues in the list
:param kwargs: Ignored arguments passed to storage API
:raises: ValidationFailed if the limit is exceeded
"""
uplimit = self._limits_conf.max_queues_per_page
if limit is not None and not (0 < limit <= uplimit):
msg = _(u'Limit must be at least 1 and no greater than {0}.')
raise ValidationFailed(msg, self._limits_conf.max_queues_per_page)
def message_length(self, content_length):
"""Restrictions on message post length.
:param content_length: Queue request's length.
:raises: ValidationFailed if the metadata is oversize.
"""
if content_length is None:
return
if content_length > self._limits_conf.max_message_size:
raise ValidationFailed(
_(u'Message collection size is too large. Max size {0}'),
self._limits_conf.max_message_size)
def validate_queue_identification(validate, req, resp, params):
"""Hook for validating the queue name and project id in requests.
The queue name validation is short-circuited if 'queue_name' does
not exist in `params`.
This hook depends on the `get_project` hook, which must be
installed upstream.
:param validate: A validator function that will
be used to check the queue name against configured
limits. functools.partial or a closure must be used to
set this first arg, and expose the remaining ones as
a Falcon hook interface.
:param req: Falcon request object
:param resp: Falcon response object
:param params: Responder params dict
"""
try:
validate(params["queue_name"], params["project_id"])
except KeyError:
# NOTE(kgriffs): queue_name not in params, so nothing to do
pass
except validation.ValidationFailed:
project = params["project_id"]
queue = params["queue_name"]
if six.PY2:
queue = queue.decode("utf-8", "replace")
LOG.debug(
u'Invalid queue name "%(queue)s" submitted for ' u"project: %(project)s",
{"queue": queue, "project": project},
)
raise falcon.HTTPBadRequest(
_(u"Invalid queue identification"),
_(u"The format of the submitted queue " u"name or project id is not valid."),
)
def on_delete(self, req, resp, project_id, queue_name):
LOG.debug(u'Queue item DELETE - queue: %(queue)s, '
u'project: %(project)s',
{'queue': queue_name, 'project': project_id})
try:
self.queue_controller.delete(queue_name, project=project_id)
except Exception as ex:
LOG.exception(ex)
description = _(u'Queue could not be deleted.')
raise wsgi_errors.HTTPServiceUnavailable(description)
resp.status = falcon.HTTP_204
def message_posting(self, messages):
"""Restrictions on a list of messages.
:param messages: A list of messages
:raises: ValidationFailed if any message has a out-of-range
TTL.
"""
if not messages:
raise ValidationFailed(_(u'No messages to enqueu.'))
for msg in messages:
self.message_content(msg)
def message_posting(self, messages):
"""Restrictions on a list of messages.
:param messages: A list of messages
:raises: ValidationFailed if any message has a out-of-range
TTL.
"""
if not messages:
raise ValidationFailed(_(u'No messages to enqueu.'))
for msg in messages:
self.message_content(msg)
def queue_identification(self, queue, project):
"""Restrictions on a project id & queue name pair.
:param queue: Name of the queue
:param project: Project id
:raises: ValidationFailed if the `name` is longer than 64
characters or contains anything other than ASCII digits and
letters, underscores, and dashes. Also raises if `project`
is not None but longer than 256 characters.
"""
if project is not None and len(project) > PROJECT_ID_MAX_LEN:
msg = _(u'Project ids may not be more than {0} characters long.')
raise ValidationFailed(msg, PROJECT_ID_MAX_LEN)
if len(queue) > QUEUE_NAME_MAX_LEN:
msg = _(u'Queue names may not be more than {0} characters long.')
raise ValidationFailed(msg, QUEUE_NAME_MAX_LEN)
if not QUEUE_NAME_REGEX.match(queue):
raise ValidationFailed(
_(u'Queue names may only contain ASCII letters, digits, '
'underscores, and dashes.'))
def queue_identification(self, queue, project):
"""Restrictions on a project id & queue name pair.
:param queue: Name of the queue
:param project: Project id
:raises: ValidationFailed if the `name` is longer than 64
characters or contains anything other than ASCII digits and
letters, underscores, and dashes. Also raises if `project`
is not None but longer than 256 characters.
"""
if project is not None and len(project) > PROJECT_ID_MAX_LEN:
msg = _(u'Project ids may not be more than {0} characters long.')
raise ValidationFailed(msg, PROJECT_ID_MAX_LEN)
if len(queue) > QUEUE_NAME_MAX_LEN:
msg = _(u'Queue names may not be more than {0} characters long.')
raise ValidationFailed(msg, QUEUE_NAME_MAX_LEN)
if not QUEUE_NAME_REGEX.match(queue):
raise ValidationFailed(
_(u'Queue names may only contain ASCII letters, digits, '
'underscores, and dashes.'))
def on_get(self, req, resp, project_id):
LOG.debug(u'Queue collection GET - project: %(project)s',
{'project': project_id})
kwargs = {}
# NOTE(kgriffs): This syntax ensures that
# we don't clobber default values with None.
req.get_param('marker', store=kwargs)
req.get_param_as_int('limit', store=kwargs)
req.get_param_as_bool('detailed', store=kwargs)
try:
self._validate.queue_listing(**kwargs)
results = self.queue_controller.list(project=project_id, **kwargs)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except Exception as ex:
LOG.exception(ex)
description = _(u'Queues could not be listed.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Buffer list of queues
queues = list(next(results))
# Check for an empty list
if len(queues) == 0:
resp.status = falcon.HTTP_204
return
# Got some. Prepare the response.
kwargs['marker'] = next(results)
for each_queue in queues:
each_queue['href'] = req.path + '/' + each_queue['name']
response_body = {
'queues': queues,
'links': [
{
'rel': 'next',
'href': req.path + falcon.to_query_str(kwargs)
}
]
}
resp.content_location = req.relative_uri
resp.body = utils.to_json(response_body)
def on_post(self, req, resp, project_id, queue_name):
LOG.debug(u'Claims collection POST - queue: %(queue)s, '
u'project: %(project)s',
{'queue': queue_name, 'project': project_id})
# Check for an explicit limit on the # of messages to claim
limit = req.get_param_as_int('limit')
claim_options = {} if limit is None else {'limit': limit}
# Read claim metadata (e.g., TTL) and raise appropriate
# HTTP errors as needed.
metadata, = wsgi_utils.filter_stream(req.stream, req.content_length,
CLAIM_POST_SPEC)
# Claim some messages
try:
self._validate.claim_creation(metadata, limit=limit)
cid, msgs = self.claim_controller.create(
queue_name,
metadata=metadata,
project=project_id,
**claim_options)
# Buffer claimed messages
# TODO(kgriffs): optimize, along with serialization (below)
resp_msgs = list(msgs)
except validation.ValidationFailed as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
except Exception as ex:
LOG.exception(ex)
description = _(u'Claim could not be created.')
raise wsgi_errors.HTTPServiceUnavailable(description)
# Serialize claimed messages, if any. This logic assumes
# the storage driver returned well-formed messages.
if len(resp_msgs) != 0:
for msg in resp_msgs:
msg['href'] = _msg_uri_from_claim(
req.path.rpartition('/')[0], msg['id'], cid)
del msg['id']
resp.location = req.path + '/' + cid
resp.body = utils.to_json(resp_msgs)
resp.status = falcon.HTTP_201
else:
resp.status = falcon.HTTP_204
def wrapper(self, *args, **kwargs):
# TODO(kgriffs): Figure out a way to not have to rely on the
# presence of `mongodb_conf`
max_attemps = self.driver.mongodb_conf.max_reconnect_attempts
sleep_sec = self.driver.mongodb_conf.reconnect_sleep
last_ex = None
for attempt in range(max_attemps):
try:
return func(self, *args, **kwargs)
break
except errors.AutoReconnect as ex:
LOG.warn(_(u'Caught AutoReconnect, retrying the '
'call to {0}').format(func))
last_ex = ex
time.sleep(sleep_sec * (2 ** attempt))
else:
LOG.error(_(u'Caught AutoReconnect, maximum attempts '
'to {0} exceeded.').format(func))
raise last_ex
def message_deletion(self, ids=None, pop=None):
"""Restrictions involving deletion of messages.
:param ids: message ids passed in by the delete request
:param pop: count of messages to be POPped
:raises: ValidationFailed if,
pop AND id params are present together
neither pop or id params are present
message count to be popped > maximum allowed
"""
if pop is not None and ids is not None:
msg = _(u'pop and id params cannot be present together in the '
'delete request.')
raise ValidationFailed(msg)
if pop is None and ids is None:
msg = _(u'The request should have either "ids" or "pop" '
'parameter in the request, to be able to delete.')
raise ValidationFailed(msg)
pop_uplimit = self._limits_conf.max_messages_per_claim_or_pop
if pop is not None and not (0 < pop <= pop_uplimit):
msg = _(u'Pop value must be at least 1 and may not '
'be greater than {0}.')
raise ValidationFailed(msg, pop_uplimit)
delete_uplimit = self._limits_conf.max_messages_per_page
if ids is not None and not (0 < len(ids) <= delete_uplimit):
msg = _(u'ids parameter should have at least 1 and not '
'greater than {0} values.')
raise ValidationFailed(msg, delete_uplimit)
def claim_updating(self, metadata):
"""Restrictions on the claim TTL.
:param metadata: The claim metadata
:raises: ValidationFailed if the TTL is out of range
"""
ttl = metadata['ttl']
if not (MIN_CLAIM_TTL <= ttl <= self._limits_conf.max_claim_ttl):
msg = _(u'The TTL for a claim may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(
msg, self._limits_conf.max_message_ttl, MIN_CLAIM_TTL)
def on_delete(self, req, resp, project_id, queue_name, message_id):
LOG.debug(
u"Messages item DELETE - message: %(message)s, " u"queue: %(queue)s, project: %(project)s",
{"message": message_id, "queue": queue_name, "project": project_id},
)
try:
self.message_controller.delete(
queue_name, message_id=message_id, project=project_id, claim=req.get_param("claim_id")
)
except storage_errors.NotPermitted as ex:
LOG.exception(ex)
title = _(u"Unable to delete")
description = _(u"This message is claimed; it cannot be " u"deleted without a valid claim_id.")
raise falcon.HTTPForbidden(title, description)
except Exception as ex:
LOG.exception(ex)
description = _(u"Message could not be deleted.")
raise wsgi_errors.HTTPServiceUnavailable(description)
# Alles guete
resp.status = falcon.HTTP_204
def claim_updating(self, metadata):
"""Restrictions on the claim TTL.
:param metadata: The claim metadata
:raises: ValidationFailed if the TTL is out of range
"""
ttl = metadata['ttl']
if not (MIN_CLAIM_TTL <= ttl <= self._limits_conf.max_claim_ttl):
msg = _(u'The TTL for a claim may not exceed {0} seconds, and '
'must be at least {1} seconds long.')
raise ValidationFailed(msg, self._limits_conf.max_message_ttl,
MIN_CLAIM_TTL)
def get_client_uuid(req):
"""Read a required Client-ID from a request.
:param req: A falcon.Request object
:raises: HTTPBadRequest if the Client-ID header is missing or
does not represent a valid UUID
:returns: A UUID object
"""
try:
return uuid.UUID(req.get_header('Client-ID', required=True))
except ValueError:
description = _(u'Malformed hexadecimal UUID.')
raise errors.HTTPBadRequestAPI(description)
def get_client_uuid(req):
"""Read a required Client-ID from a request.
:param req: A falcon.Request object
:raises: HTTPBadRequest if the Client-ID header is missing or
does not represent a valid UUID
:returns: A UUID object
"""
try:
return uuid.UUID(req.get_header('Client-ID', required=True))
except ValueError:
description = _(u'Malformed hexadecimal UUID.')
raise errors.HTTPBadRequestAPI(description)
def on_put(self, req, resp, project_id, queue_name):
LOG.debug(u'Queue item PUT - queue: %(queue)s, '
u'project: %(project)s',
{'queue': queue_name, 'project': project_id})
try:
created = self.queue_controller.create(
queue_name, project=project_id)
except Exception as ex:
LOG.exception(ex)
description = _(u'Queue could not be created.')
raise wsgi_errors.HTTPServiceUnavailable(description)
resp.status = falcon.HTTP_201 if created else falcon.HTTP_204
resp.location = req.path
def on_delete(self, req, resp, project_id, queue_name, claim_id):
LOG.debug(u'Claim item DELETE - claim: %(claim_id)s, '
u'queue: %(queue_name)s, project: %(project_id)s' %
{'queue_name': queue_name,
'project_id': project_id,
'claim_id': claim_id})
try:
self.claim_controller.delete(queue_name,
claim_id=claim_id,
project=project_id)
resp.status = falcon.HTTP_204
except Exception as ex:
LOG.exception(ex)
description = _(u'Claim could not be deleted.')
raise wsgi_errors.HTTPServiceUnavailable(description)
def get_schema(self, operation):
"""Returns the schema for an operation
:param operation: Operation for which params need
to be validated.
:type operation: `six.text_type`
:returns: Operation's schema
:rtype: dict
:raises: `errors.InvalidOperation` if the operation
does not exist
"""
try:
return self.schema[operation]
except KeyError:
# TODO(flaper87): gettext support
msg = _('{0} is not a valid operation').format(operation)
raise errors.InvalidOperation(msg)