def on_get(self, request, response):
project = helpers.get_project(request)
key = 'qs.%s' % project
if not self.client.exists(key):
response.status = falcon.HTTP_204
return
resp = {}
for q in self.client.lrange(key, 0, -1):
hkey = 'q.%s.%s' % (project, q.decode('utf8'))
queue = q.decode('utf8')
h, n, m = self.client.hmget(hkey, ['h', 'n', 'm'])
if not all([h, n]):
continue
resp[queue] = {
'host': h.decode('utf8'),
'name': n.decode('utf8')
}
resp[queue]['metadata'] = msgpack.loads(m) if m else {}
if not resp:
response.status = falcon.HTTP_204
return
response.status = falcon.HTTP_200
response.body = json.dumps(resp, ensure_ascii=False)
def on_put(self, request, response, queue):
project = helpers.get_project(request)
data = request.stream.read()
# NOTE(cpp-cabrera): This is a hack to preserve the metadata
request.stream = io.BytesIO(data)
resp = self.forward(request, response, queue)
if resp.ok:
self._catalogue.update_metadata(project, queue, json.loads(data))
def on_delete(self, request, response, queue):
project = helpers.get_project(request)
LOG.debug('DELETE queue - project/queue: {0}/{1}'.format(
project, queue))
resp = self.forward(request, response, queue)
# avoid deleting a queue if the request is bad
if resp.ok:
self._catalogue.delete(project, queue)
lookup.invalidate_entry(project, queue, self._cache)
def on_put(self, request, response, queue):
project = helpers.get_project(request)
data = request.stream.read()
# NOTE(cpp-cabrera): This is a hack to preserve the metadata
request.stream = io.BytesIO(data)
resp = self.forward(request, response, queue)
if resp.ok:
self._catalogue.update_metadata(project, queue, json.loads(data))
def on_delete(self, request, response, queue):
project = helpers.get_project(request)
LOG.debug('DELETE queue - project/queue: {0}/{1}'.format(
project, queue
))
resp = self.forward(request, response, queue)
# avoid deleting a queue if the request is bad
if resp.ok:
self._catalogue.delete(project, queue)
lookup.invalidate_entry(project, queue, self._cache)
def on_get(self, request, response):
project = helpers.get_project(request)
LOG.debug("LIST catalogue - project: {0}".format(project))
resp = list(self._catalogue.list(project))
if not resp:
response.status = falcon.HTTP_204
return
response.status = falcon.HTTP_200
response.body = json.dumps(resp, ensure_ascii=False)
def on_get(self, request, response, queue):
project = helpers.get_project(request)
LOG.debug("GET catalogue - project/queue: {0}/{1}".format(project, queue))
entry = None
try:
entry = self._catalogue.get(project, queue)
except exceptions.EntryNotFound:
LOG.debug("Entry not found")
raise falcon.HTTPNotFound()
response.status = falcon.HTTP_200
response.body = json.dumps(entry, ensure_ascii=False)
def on_delete(self, request, response, queue):
key = self._make_key(request, queue)
project = helpers.get_project(request)
resp = helpers.forward(self.client, request, queue)
response.set_headers(resp.headers)
response.status = http.status(resp.status_code)
# avoid deleting a queue if the request is bad
if not resp.ok:
self.client.hdel(key, queue)
self.client.lrem('qs.%s' % project, 1, queue)
def on_get(self, request, response):
project = helpers.get_project(request)
LOG.debug('LIST catalogue - project: {0}'.format(project))
resp = list(self._catalogue.list(project))
if not resp:
response.status = falcon.HTTP_204
return
response.status = falcon.HTTP_200
response.body = json.dumps(resp, ensure_ascii=False)
def on_get(self, request, response, queue):
key = 'q.%s.%s' % (helpers.get_project(request), queue)
if not self.client.exists(key):
raise falcon.HTTPNotFound()
h, n, m = self.client.hmget(key, ['h', 'n', 'm'])
resp = {
'name': n.decode('utf8'),
'host': h.decode('utf8'),
}
resp['metadata'] = msgpack.loads(m) if m else {}
response.status = falcon.HTTP_200
response.body = json.dumps(resp, ensure_ascii=False)
def on_get(self, request, response, queue):
project = helpers.get_project(request)
LOG.debug('GET catalogue - project/queue: {0}/{1}'.format(
project, queue
))
entry = None
try:
entry = self._catalogue.get(project, queue)
except exceptions.EntryNotFound:
LOG.debug('Entry not found')
raise falcon.HTTPNotFound()
response.status = falcon.HTTP_200
response.body = json.dumps(entry, ensure_ascii=False)
def on_put(self, request, response, queue):
key = self._make_key(request, queue)
if not self.client.exists(key):
raise falcon.HTTPNotFound()
resp = helpers.forward(self.client, request, queue)
response.status = http.status(resp.status_code)
response.body = resp.content
if resp.ok:
project = helpers.get_project(request)
host = helpers.get_host_by_project_and_queue(self.client,
project, queue)
resp = requests.get(host + '/v1/queues/%s/metadata' % queue)
self.client.hset(key, 'm', msgpack.dumps(resp.json()))
def on_put(self, request, response, queue):
"""Create a queue in the catalogue, then forwards to marconi.
This is the only time marconi proxy ever needs to select a
partition for a queue. The association is created in the
catalogue. This should also be the only time
partition.weighted_select is ever called.
:raises: HTTPInternalServerError - if no partitions are registered
"""
project = helpers.get_project(request)
# NOTE(cpp-cabrera): if we've already registered a queue,
# don't try to create it again, because it will duplicate it
# across partitions.
#
# There exists a race condition here, but it is benign. It's
# possible that after the existence check has succeeded,
# another request may succeed in DELETEing a queue. In this
# scenario, the queue will be recreated on another partition,
# which is reasonable, since the user meant to both DELETE and
# PUT That queue.
if lookup.exists(project, queue,
self._catalogue, self._cache):
response.status = falcon.HTTP_204
return
target = partition.weighted_select(self._partitions.list())
if target is None:
LOG.error('No partitions registered')
raise falcon.HTTPInternalServerError(
"No partitions registered",
"Contact the system administrator for more details."
)
host = target['hosts'][0]
resp = helpers.forward(host, request)
# NOTE(cpp-cabrera): only catalogue a queue if it was created
if resp.status_code == 201:
self._catalogue.insert(project, queue, target['name'],
host)
response.set_headers(helpers.capitalized(resp.headers))
response.status = http.status(resp.status_code)
response.body = resp.content
def on_get(self, request, response):
project = helpers.get_project(request)
LOG.debug('LIST queues - project: {0}'.format(project))
kwargs = {}
request.get_param('marker', store=kwargs)
request.get_param_as_int('limit', store=kwargs)
request.get_param_as_bool('detailed', store=kwargs)
resp = collections.defaultdict(list)
limit = kwargs.get('limit',
STORAGE_LIMITS.default_queue_paging)
try:
validate.queue_listing(limit=limit)
except validate.ValidationFailed as ex:
raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
for queue in self._catalogue.list(project):
queue_name = queue['name']
if queue_name < kwargs.get('marker', ''):
continue
entry = {
'href': request.path + '/' + queue_name,
'name': queue_name
}
if kwargs.get('detailed', None):
entry['metadata'] = queue['metadata']
resp['queues'].append(entry)
kwargs['marker'] = queue_name
if len(resp['queues']) == limit:
break
if not resp:
LOG.debug('LIST queues - no queues found')
response.status = falcon.HTTP_204
return
resp['links'].append({
'rel': 'next',
'href': request.path + falcon.to_query_str(kwargs)
})
response.content_location = request.relative_uri
response.body = json.dumps(resp, ensure_ascii=False)
def on_get(self, request, response):
project = helpers.get_project(request)
LOG.debug('LIST queues - project: {0}'.format(project))
kwargs = {}
request.get_param('marker', store=kwargs)
request.get_param_as_int('limit', store=kwargs)
request.get_param_as_bool('detailed', store=kwargs)
resp = collections.defaultdict(list)
limit = kwargs.get('limit', STORAGE_LIMITS.default_queue_paging)
try:
validate.queue_listing(limit=limit)
except validate.ValidationFailed as ex:
raise wsgi_exceptions.HTTPBadRequestAPI(six.text_type(ex))
for queue in self._catalogue.list(project):
queue_name = queue['name']
if queue_name < kwargs.get('marker', ''):
continue
entry = {
'href': request.path + '/' + queue_name,
'name': queue_name
}
if kwargs.get('detailed', None):
entry['metadata'] = queue['metadata']
resp['queues'].append(entry)
kwargs['marker'] = queue_name
if len(resp['queues']) == limit:
break
if not resp:
LOG.debug('LIST queues - no queues found')
response.status = falcon.HTTP_204
return
resp['links'].append({
'rel': 'next',
'href': request.path + falcon.to_query_str(kwargs)
})
response.content_location = request.relative_uri
response.body = json.dumps(resp, ensure_ascii=False)
def on_get(self, request, response):
project = helpers.get_project(request)
key = 'qs.%s' % project
if not self.client.exists(key):
response.status = falcon.HTTP_204
return
kwargs = {}
request.get_param('marker', store=kwargs)
request.get_param_as_int('limit', store=kwargs)
request.get_param_as_bool('detailed', store=kwargs)
resp = collections.defaultdict(list)
for q in sorted(self.client.lrange(key, 0, -1)):
queue = q.decode('utf8')
if queue < kwargs.get('marker', 0):
continue
entry = {
'href': request.path + '/' + queue,
'name': queue
}
if kwargs.get('detailed', None):
qkey = 'q.%s.%s' % (project, queue)
data = self.client.hget(qkey, 'm')
metadata = msgpack.loads(data)
entry['metadata'] = metadata
resp['queues'].append(entry)
kwargs['marker'] = queue
if len(resp['queues']) == kwargs.get('limit', None):
break
if not resp:
response.status = falcon.HTTP_204
return
resp['links'].append({
'rel': 'next',
'href': request.path + falcon.to_query_str(kwargs)
})
response.content_location = request.relative_uri
response.body = json.dumps(resp, ensure_ascii=False)
def on_put(self, request, response, queue):
"""Create a queue in the catalogue, then forwards to marconi.
This is the only time marconi proxy ever needs to select a
partition for a queue. The association is created in the
catalogue. This should also be the only time
partition.weighted_select is ever called.
:raises: HTTPInternalServerError - if no partitions are registered
"""
project = helpers.get_project(request)
# NOTE(cpp-cabrera): if we've already registered a queue,
# don't try to create it again, because it will duplicate it
# across partitions.
#
# There exists a race condition here, but it is benign. It's
# possible that after the existence check has succeeded,
# another request may succeed in DELETEing a queue. In this
# scenario, the queue will be recreated on another partition,
# which is reasonable, since the user meant to both DELETE and
# PUT That queue.
if lookup.exists(project, queue, self._catalogue, self._cache):
response.status = falcon.HTTP_204
return
target = partition.weighted_select(self._partitions.list())
if target is None:
LOG.error('No partitions registered')
raise falcon.HTTPInternalServerError(
"No partitions registered",
"Contact the system administrator for more details.")
host = target['hosts'][0]
resp = helpers.forward(host, request)
# NOTE(cpp-cabrera): only catalogue a queue if it was created
if resp.status_code == 201:
self._catalogue.insert(project, queue, target['name'], host)
response.set_headers(helpers.capitalized(resp.headers))
response.status = http.status(resp.status_code)
response.body = resp.content
def forward(self, request, response, queue, **kwargs):
"""Forwards requests in a selector-driven fashion."""
project = helpers.get_project(request)
LOG.debug('FORWARD - project/queue: {0}/{1}'.format(
project, queue
))
partition = lookup.partition(project, queue,
self._catalogue,
self._cache)
# NOTE(cpp-cabrera): we tried to look up a catalogue
# entry and it failed. This happens if the associated
# queue doesn't exist under that project.
if not partition:
LOG.debug('Catalogue entry not found')
raise falcon.HTTPNotFound()
hosts = lookup.hosts(partition, self._partitions, self._cache)
# NOTE(cpp-cabrera): we tried to look up a partition, and it
# failed. This only happens if a partition is deleted from
# the primary store between here and the last call.
if not hosts:
LOG.debug('Partition not found')
raise falcon.HTTPNotFound()
# round robin to choose the desired host
host = self._selector.next(partition, hosts)
# send the request, update the response
resp = helpers.forward(host, request)
# NOTE(zyuan): normalize the lower-case header from
# `requests` to Caml-Case and forward the headers back
response.set_headers(helpers.capitalized(resp.headers))
response.status = http.status(resp.status_code)
response.body = resp.content
# NOTE(cpp-cabrera): in case responder must do more afterwards
return resp
def on_put(self, request, response, queue):
key = self._make_key(request, queue)
project = helpers.get_project(request)
if self.client.exists(key):
response.status = falcon.HTTP_204
return
partition = node.weighted_select(self.client)
host = node.round_robin(self.client, partition)
url = '{host}/v1/queues/{queue}'.format(host=host, queue=queue)
resp = requests.put(url, headers=request._headers)
# NOTE(cpp-cabrera): only catalogue a queue if a request is good
if resp.ok:
self.client.hmset(key, {
'h': host,
'n': queue
})
self.client.rpush('qs.%s' % project, queue)
response.status = http.status(resp.status_code)
response.body = resp.content
def forward(self, request, response, queue, **kwargs):
"""Forwards requests in a selector-driven fashion."""
project = helpers.get_project(request)
LOG.debug('FORWARD - project/queue: {0}/{1}'.format(project, queue))
partition = lookup.partition(project, queue, self._catalogue,
self._cache)
# NOTE(cpp-cabrera): we tried to look up a catalogue
# entry and it failed. This happens if the associated
# queue doesn't exist under that project.
if not partition:
LOG.debug('Catalogue entry not found')
raise falcon.HTTPNotFound()
hosts = lookup.hosts(partition, self._partitions, self._cache)
# NOTE(cpp-cabrera): we tried to look up a partition, and it
# failed. This only happens if a partition is deleted from
# the primary store between here and the last call.
if not hosts:
LOG.debug('Partition not found')
raise falcon.HTTPNotFound()
# round robin to choose the desired host
host = self._selector.next(partition, hosts)
# send the request, update the response
resp = helpers.forward(host, request)
# NOTE(zyuan): normalize the lower-case header from
# `requests` to Caml-Case and forward the headers back
response.set_headers(helpers.capitalized(resp.headers))
response.status = http.status(resp.status_code)
response.body = resp.content
# NOTE(cpp-cabrera): in case responder must do more afterwards
return resp
def _make_key(self, request, queue):
project = helpers.get_project(request)
return 'q.%s.%s' % (project, queue)