def _create_msg(self, queue, msg, client_uuid, project):
slug = str(uuid.uuid1())
contents = jsonutils.dumps(
{'body': msg.get('body', {}), 'claim_id': None,
'ttl': msg['ttl'], 'claim_count': 0})
try:
self._client.put_object(
utils._message_container(queue, project),
slug,
contents=contents,
content_type='application/json',
headers={
'x-object-meta-clientid': str(client_uuid),
'x-delete-after': msg['ttl']})
except swiftclient.ClientException as exc:
# NOTE(flwang): To avoid check the queue existence each time when
# posting messages, let's catch the 404 and create the 'queue'
# on demand.
if exc.http_status == 404:
self._client.put_container(utils._message_container(queue,
project))
self._client.put_object(
utils._message_container(queue, project),
slug,
contents=contents,
content_type='application/json',
headers={
'x-object-meta-clientid': str(client_uuid),
'x-delete-after': msg['ttl']})
return slug
def _delete(self, queue, message_id, project=None):
try:
self._client.delete_object(
utils._message_container(queue, project), message_id)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
def _delete_queue_messages(self, queue, project, pipe):
"""Method to remove all the messages belonging to a queue.
Will be referenced from the QueueController.
The pipe to execute deletion will be passed from the QueueController
executing the operation.
"""
container = utils._message_container(queue, project)
remaining = True
key = ''
while remaining:
headers, objects = self._client.get_container(container,
limit=1000,
marker=key)
if not objects:
return
remaining = len(objects) == 1000
key = objects[-1]['name']
for o in objects:
try:
self._client.delete_object(container, o['name'])
except swiftclient.ClientException as exc:
if exc.http_status == 404:
continue
raise
def _delete(self, queue, message_id, project=None):
try:
self._client.delete_object(
utils._message_container(queue, project), message_id)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
def _delete_queue_messages(self, queue, project, pipe):
"""Method to remove all the messages belonging to a queue.
Will be referenced from the QueueController.
The pipe to execute deletion will be passed from the QueueController
executing the operation.
"""
container = utils._message_container(queue, project)
remaining = True
key = ''
while remaining:
headers, objects = self._client.get_container(container,
limit=1000,
marker=key)
if not objects:
return
remaining = len(objects) == 1000
key = objects[-1]['name']
for o in objects:
try:
self._client.delete_object(container, o['name'])
except swiftclient.ClientException as exc:
if exc.http_status == 404:
continue
raise
def delete(self, queue, claim_id, project=None):
message_ctrl = self.driver.message_controller
try:
header, obj = self._client.get_object(
utils._claim_container(queue, project),
claim_id)
for msg_id in jsonutils.loads(obj):
try:
headers, msg = message_ctrl._find_message(queue, msg_id,
project)
except errors.MessageDoesNotExist:
continue
md5 = hashlib.md5()
md5.update(msg.encode('utf-8'))
md5 = md5.hexdigest()
msg = jsonutils.loads(msg)
content = jsonutils.dumps(
{'body': msg['body'], 'claim_id': None, 'ttl': msg['ttl']})
client_id = headers['x-object-meta-clientid']
self._client.put_object(
utils._message_container(queue, project),
msg_id,
content,
content_type='application/json',
headers={'x-object-meta-clientid': client_id,
'if-match': md5,
'x-delete-at': headers['x-delete-at']})
self._client.delete_object(
utils._claim_container(queue, project),
claim_id)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
def _create_msg(self, queue, msg, client_uuid, project):
slug = str(uuid.uuid1())
now = timeutils.utcnow_ts()
message = {
'body': msg.get('body', {}),
'claim_id': None,
'ttl': msg['ttl'],
'claim_count': 0,
'delay_expires': now + msg.get('delay', 0)
}
if self.driver.conf.enable_checksum:
message['checksum'] = s_utils.get_checksum(msg.get('body', None))
contents = jsonutils.dumps(message)
utils._put_or_create_container(self._client,
utils._message_container(
queue, project),
slug,
contents=contents,
content_type='application/json',
headers={
'x-object-meta-clientid':
str(client_uuid),
'x-delete-after':
msg['ttl']
})
return slug
def _list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None,
include_claimed=False, sort=1):
"""List messages in the queue, oldest first(ish)
Time ordering and message inclusion in lists are soft, there is no
global order and times are based on the UTC time of the zaqar-api
server that the message was created from.
Here be consistency dragons.
"""
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
client = self._client
container = utils._message_container(queue, project)
query_string = None
if sort == -1:
query_string = 'reverse=on'
try:
_, objects = client.get_container(
container,
marker=marker,
# list 2x the objects because some listing items may have
# expired
limit=limit * 2,
query_string=query_string)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueDoesNotExist(queue, project)
raise
def is_claimed(msg, headers):
if include_claimed or msg['claim_id'] is None:
return False
claim_obj = self.driver.claim_controller._get(
queue, msg['claim_id'], project)
return claim_obj is not None and claim_obj['ttl'] > 0
def is_echo(msg, headers):
if echo:
return False
return headers['x-object-meta-clientid'] == str(client_uuid)
filters = [
is_echo,
is_claimed,
]
marker = {}
get_object = functools.partial(client.get_object, container)
list_objects = functools.partial(client.get_container, container,
limit=limit * 2,
query_string=query_string)
yield utils._filter_messages(objects, filters, marker, get_object,
list_objects, limit=limit)
yield marker and marker['next']
def _find_message(self, queue, message_id, project):
try:
return self._client.get_object(
utils._message_container(queue, project), message_id)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.MessageDoesNotExist(message_id, queue, project)
else:
raise
def _find_message(self, queue, message_id, project):
try:
return self._client.get_object(
utils._message_container(queue, project), message_id)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.MessageDoesNotExist(message_id, queue, project)
else:
raise
def exists(self, queue, project=None):
try:
self._client.head_container(utils._message_container(queue,
project))
except swiftclient.ClientException as exc:
if exc.http_status == 404:
return False
raise
else:
return True
def exists(self, topic, project=None):
try:
self._client.head_container(utils._message_container(topic,
project))
except swiftclient.ClientException as exc:
if exc.http_status == 404:
return False
raise
else:
return True
def stats(self, name, project=None):
if not self._queue_ctrl.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = 0
claimed = 0
container = utils._message_container(name, project)
try:
_, objects = self._client.get_container(container)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueIsEmpty(name, project)
newest = None
oldest = None
now = timeutils.utcnow_ts(True)
for obj in objects:
try:
headers = self._client.head_object(container, obj['name'])
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
else:
created = float(headers['x-timestamp'])
created_iso = datetime.datetime.utcfromtimestamp(
created).strftime('%Y-%m-%dT%H:%M:%SZ')
newest = {
'id': obj['name'],
'age': now - created,
'created': created_iso
}
if oldest is None:
oldest = copy.deepcopy(newest)
total += 1
if headers.get('x-object-meta-claimid'):
claimed += 1
msg_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if newest is not None:
msg_stats['newest'] = newest
msg_stats['oldest'] = oldest
return {'messages': msg_stats}
def stats(self, name, project=None):
if not self._queue_ctrl.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = 0
claimed = 0
container = utils._message_container(name, project)
try:
_, objects = self._client.get_container(container)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueIsEmpty(name, project)
newest = None
oldest = None
now = timeutils.utcnow_ts(True)
for obj in objects:
try:
headers = self._client.head_object(container, obj['name'])
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
else:
created = float(headers['x-timestamp'])
created_iso = datetime.datetime.utcfromtimestamp(
created).strftime('%Y-%m-%dT%H:%M:%SZ')
newest = {
'id': obj['name'],
'age': now - created,
'created': created_iso}
if oldest is None:
oldest = copy.deepcopy(newest)
total += 1
if headers.get('x-object-meta-claimid'):
claimed += 1
msg_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if newest is not None:
msg_stats['newest'] = newest
msg_stats['oldest'] = oldest
return {'messages': msg_stats}
def delete(self, name, project=None):
for container in [utils._message_container(name, project)]:
try:
headers, objects = self._client.get_container(container)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
else:
for obj in objects:
try:
self._client.delete_object(container, obj['name'])
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
try:
self._client.delete_container(container)
except swiftclient.ClientException as exc:
if exc.http_status not in (404, 409):
raise
def _create_msg(self, queue, msg, client_uuid, project):
slug = str(uuid.uuid1())
now = timeutils.utcnow_ts()
message = {'body': msg.get('body', {}), 'claim_id': None,
'ttl': msg['ttl'], 'claim_count': 0,
'delay_expires': now + msg.get('delay', 0)}
if self.driver.conf.enable_checksum:
message['checksum'] = s_utils.get_checksum(msg.get('body', None))
contents = jsonutils.dumps(message)
utils._put_or_create_container(
self._client,
utils._message_container(queue, project),
slug,
contents=contents,
content_type='application/json',
headers={
'x-object-meta-clientid': str(client_uuid),
'x-delete-after': msg['ttl']})
return slug
def delete(self, queue, claim_id, project=None):
message_ctrl = self.driver.message_controller
try:
header, obj = self._client.get_object(
utils._claim_container(queue, project), claim_id)
for msg_id in jsonutils.loads(obj):
try:
headers, msg = message_ctrl._find_message(
queue, msg_id, project)
except errors.MessageDoesNotExist:
continue
md5 = hashlib.md5()
md5.update(msg.encode('utf-8'))
md5 = md5.hexdigest()
msg = jsonutils.loads(msg)
content = jsonutils.dumps({
'body': msg['body'],
'claim_id': None,
'ttl': msg['ttl']
})
client_id = headers['x-object-meta-clientid']
self._client.put_object(
utils._message_container(queue, project),
msg_id,
content,
content_type='application/json',
headers={
'x-object-meta-clientid': client_id,
'if-match': md5,
'x-delete-at': headers['x-delete-at']
})
self._client.delete_object(utils._claim_container(queue, project),
claim_id)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
def create(self, name, metadata=None, project=None):
self._client.put_container(utils._message_container(name, project))
def create(self, name, metadata=None, project=None):
self._client.put_container(utils._message_container(name, project))
def create(self, queue, metadata, project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
message_ctrl = self.driver.message_controller
queue_ctrl = self.driver.queue_controller
try:
queue_meta = queue_ctrl.get_metadata(queue, project=project)
except errors.QueueDoesNotExist:
return None, iter([])
ttl = metadata['ttl']
grace = metadata['grace']
msg_ts = ttl + grace
claim_id = uuidutils.generate_uuid()
dlq = True if ('_max_claim_count' in queue_meta and
'_dead_letter_queue' in queue_meta) else False
include_delayed = False if queue_meta.get('_default_message_delay',
0) else True
messages, marker = message_ctrl._list(queue, project, limit=limit,
include_claimed=False,
include_delayed=include_delayed)
claimed = []
for msg in messages:
claim_count = msg.get('claim_count', 0)
md5 = hashlib.md5()
md5.update(
jsonutils.dumps(
{'body': msg['body'], 'claim_id': None,
'ttl': msg['ttl'],
'claim_count': claim_count}).encode('utf-8'))
md5 = md5.hexdigest()
msg_ttl = max(msg['ttl'], msg_ts)
move_to_dlq = False
if dlq:
if claim_count < queue_meta['_max_claim_count']:
# Check if the message's claim count has exceeded the
# max claim count defined in the queue, if not ,
# Save the new max claim count for message
claim_count = claim_count + 1
else:
# if the message's claim count has exceeded the
# max claim count defined in the queue, move the
# message to the dead letter queue.
# NOTE: We're moving message by changing the
# project info directly. That means, the queue and dead
# letter queue must be created on the same pool.
dlq_ttl = queue_meta.get("_dead_letter_queue_messages_ttl")
move_to_dlq = True
if dlq_ttl:
msg_ttl = dlq_ttl
content = jsonutils.dumps(
{'body': msg['body'], 'claim_id': claim_id,
'ttl': msg_ttl,
'claim_count': claim_count})
if move_to_dlq:
dead_letter_queue = queue_meta.get("_dead_letter_queue")
utils._put_or_create_container(
self._client,
utils._message_container(dead_letter_queue, project),
msg['id'],
content,
content_type='application/json',
headers={'x-object-meta-clientid': msg['client_uuid'],
'if-match': md5,
'x-object-meta-claimid': claim_id,
'x-delete-after': msg_ttl})
message_ctrl._delete(queue, msg['id'], project)
else:
try:
self._client.put_object(
utils._message_container(queue, project),
msg['id'],
content,
content_type='application/json',
headers={'x-object-meta-clientid': msg['client_uuid'],
'if-match': md5,
'x-object-meta-claimid': claim_id,
'x-delete-after': msg_ttl})
except swiftclient.ClientException as exc:
if exc.http_status == 412:
continue
raise
else:
msg['claim_id'] = claim_id
msg['ttl'] = msg_ttl
msg['claim_count'] = claim_count
claimed.append(msg)
utils._put_or_create_container(
self._client,
utils._claim_container(queue, project),
claim_id,
jsonutils.dumps([msg['id'] for msg in claimed]),
content_type='application/json',
headers={'x-delete-after': ttl}
)
return claim_id, claimed
def _list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None,
include_claimed=False, include_delayed=False,
sort=1):
"""List messages in the queue, oldest first(ish)
Time ordering and message inclusion in lists are soft, there is no
global order and times are based on the UTC time of the zaqar-api
server that the message was created from.
Here be consistency dragons.
"""
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
client = self._client
container = utils._message_container(queue, project)
query_string = None
if sort == -1:
query_string = 'reverse=on'
try:
_, objects = client.get_container(
container,
marker=marker,
# list 2x the objects because some listing items may have
# expired
limit=limit * 2,
query_string=query_string)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueDoesNotExist(queue, project)
raise
def is_claimed(msg, headers):
if include_claimed or msg['claim_id'] is None:
return False
claim_obj = self.driver.claim_controller._get(
queue, msg['claim_id'], project)
return claim_obj is not None and claim_obj['ttl'] > 0
def is_delayed(msg, headers):
if include_delayed:
return False
now = timeutils.utcnow_ts()
return msg.get('delay_expires', 0) > now
def is_echo(msg, headers):
if echo:
return False
return headers['x-object-meta-clientid'] == str(client_uuid)
filters = [
is_echo,
is_claimed,
is_delayed,
]
marker = {}
get_object = functools.partial(client.get_object, container)
list_objects = functools.partial(client.get_container, container,
limit=limit * 2,
query_string=query_string)
yield utils._filter_messages(objects, filters, marker, get_object,
list_objects, limit=limit)
yield marker and marker['next']
def create(self,
queue,
metadata,
project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
ttl = metadata['ttl']
grace = metadata['grace']
msg_ts = ttl + grace
claim_id = uuidutils.generate_uuid()
messages, marker = self._message_ctrl._list(queue,
project,
limit=limit,
include_claimed=False)
claimed = []
for msg in messages:
md5 = hashlib.md5()
md5.update(
jsonutils.dumps({
'body': msg['body'],
'claim_id': None,
'ttl': msg['ttl']
}))
md5 = md5.hexdigest()
msg_ttl = max(msg['ttl'], msg_ts)
content = jsonutils.dumps({
'body': msg['body'],
'claim_id': claim_id,
'ttl': msg_ttl
})
try:
self._client.put_object(utils._message_container(
queue, project),
msg['id'],
content,
content_type='application/json',
headers={
'x-object-meta-clientid':
msg['client_uuid'],
'if-match':
md5,
'x-object-meta-claimid':
claim_id,
'x-delete-after':
msg_ttl
})
except swiftclient.ClientException as exc:
if exc.http_status == 412:
continue
raise
else:
msg['claim_id'] = claim_id
msg['ttl'] = msg_ttl
claimed.append(msg)
utils._put_or_create_container(self._client,
utils._claim_container(queue, project),
claim_id,
jsonutils.dumps(
[msg['id'] for msg in claimed]),
content_type='application/json',
headers={'x-delete-after': ttl})
return claim_id, claimed
def create(self,
queue,
metadata,
project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
message_ctrl = self.driver.message_controller
queue_ctrl = self.driver.queue_controller
try:
queue_meta = queue_ctrl.get_metadata(queue, project=project)
except errors.QueueDoesNotExist:
return None, iter([])
ttl = metadata['ttl']
grace = metadata['grace']
msg_ts = ttl + grace
claim_id = uuidutils.generate_uuid()
dlq = True if ('_max_claim_count' in queue_meta
and '_dead_letter_queue' in queue_meta) else False
include_delayed = False if queue_meta.get('_default_message_delay',
0) else True
messages, marker = message_ctrl._list(queue,
project,
limit=limit,
include_claimed=False,
include_delayed=include_delayed)
claimed = []
for msg in messages:
claim_count = msg.get('claim_count', 0)
md5 = hashlib.md5()
md5.update(
jsonutils.dumps({
'body': msg['body'],
'claim_id': None,
'ttl': msg['ttl'],
'claim_count': claim_count
}).encode('utf-8'))
md5 = md5.hexdigest()
msg_ttl = max(msg['ttl'], msg_ts)
move_to_dlq = False
if dlq:
if claim_count < queue_meta['_max_claim_count']:
# Check if the message's claim count has exceeded the
# max claim count defined in the queue, if not ,
# Save the new max claim count for message
claim_count = claim_count + 1
else:
# if the message's claim count has exceeded the
# max claim count defined in the queue, move the
# message to the dead letter queue.
# NOTE: We're moving message by changing the
# project info directly. That means, the queue and dead
# letter queue must be created on the same pool.
dlq_ttl = queue_meta.get("_dead_letter_queue_messages_ttl")
move_to_dlq = True
if dlq_ttl:
msg_ttl = dlq_ttl
content = jsonutils.dumps({
'body': msg['body'],
'claim_id': claim_id,
'ttl': msg_ttl,
'claim_count': claim_count
})
if move_to_dlq:
dead_letter_queue = queue_meta.get("_dead_letter_queue")
utils._put_or_create_container(self._client,
utils._message_container(
dead_letter_queue, project),
msg['id'],
content,
content_type='application/json',
headers={
'x-object-meta-clientid':
msg['client_uuid'],
'if-match':
md5,
'x-object-meta-claimid':
claim_id,
'x-delete-after':
msg_ttl
})
message_ctrl._delete(queue, msg['id'], project)
else:
try:
self._client.put_object(utils._message_container(
queue, project),
msg['id'],
content,
content_type='application/json',
headers={
'x-object-meta-clientid':
msg['client_uuid'],
'if-match':
md5,
'x-object-meta-claimid':
claim_id,
'x-delete-after':
msg_ttl
})
except swiftclient.ClientException as exc:
if exc.http_status == 412:
continue
raise
else:
msg['claim_id'] = claim_id
msg['ttl'] = msg_ttl
msg['claim_count'] = claim_count
claimed.append(msg)
utils._put_or_create_container(self._client,
utils._claim_container(queue, project),
claim_id,
jsonutils.dumps(
[msg['id'] for msg in claimed]),
content_type='application/json',
headers={'x-delete-after': ttl})
return claim_id, claimed