def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = self._client.hgetall(q_id)
claimed = int(q_info['cl'])
total = int(q_info['c'])
msg_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts()
message_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total
}
try:
newest = msg_ctrl.first(name, project, 1)
oldest = msg_ctrl.first(name, project, -1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = utils.stat_message(newest, now)
message_stats['oldest'] = utils.stat_message(oldest, now)
return {'messages': message_stats}
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
metadata = self._get_queue_info(q_id, 'm')
return {} if utils.is_metadata_empty(metadata) else metadata
def get_qid(driver, queue, project):
sel = sa.sql.select([tables.Queues.c.id],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == queue))
try:
return driver.get(sel)[0]
except NoResult:
raise errors.QueueDoesNotExist(queue, project)
def set_metadata(self, name, metadata, project=None):
target = self._lookup(name, project)
if target:
control = target.queue_controller
return control.set_metadata(name,
metadata=metadata,
project=project)
raise errors.QueueDoesNotExist(name, project)
def delete(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_name = utils.scope_queue_name(name, project)
project_bkt = self._get_project_bkt(project)
# Delete ensures the removal of the queue from the
# project bucket.
project_bkt.delete(q_name)
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_name = utils.scope_queue_name(name, project)
project_bkt = self._get_project_bkt(project)
q_info = project_bkt.get(q_name).data
return q_info['m']
def post(self, queue, project, messages, client_uuid):
target = self._lookup(queue, project)
if target:
control = target.message_controller
return control.post(queue,
project=project,
messages=messages,
client_uuid=client_uuid)
raise errors.QueueDoesNotExist(project, queue)
def get_qid(driver, queue, project):
try:
return driver.get(
'''
select id from Queues
where project = ? and name = ?''', project, queue)[0]
except NoResult:
raise errors.QueueDoesNotExist(queue, project)
def get(self, queue, message_id, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
if not self._exists(queue, project, message_id):
raise errors.MessageDoesNotExist(message_id, queue, project)
msg = self._get(message_id)
return utils.basic_message(msg, timeutils.utcnow_ts())
def list(self,
queue,
project=None,
marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=True,
client_uuid=None,
include_claimed=False):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
client = self._client
start = client.zrank(messages_set_id, marker) or 0
# In a pooled environment, the default values are not being set.
limit = limit or storage.DEFAULT_MESSAGES_PER_CLAIM
message_ids = client.zrange(messages_set_id, start, start + limit)
filters = collections.defaultdict(dict)
# Build a list of filters for checking the following:
# 1. Message is claimed.
# 2. echo message to the client.
if not include_claimed:
filters['claimed_filter']['f'] = utils.msg_claimed_filter
filters['claimed_filter']['f.v'] = timeutils.utcnow_ts()
if not echo:
filters['echo_filter']['f'] = utils.msg_echo_filter
filters['echo_filter']['f.v'] = client_uuid
marker = {}
def _it(message_ids, filters={}):
"""Create a filtered iterator of messages.
The function accepts a list of filters to be filtered
before the the message can be included as a part of the reply.
"""
now = timeutils.utcnow_ts()
for message_id in message_ids:
message = client.hgetall(message_id)
if message:
for filter in six.itervalues(filters):
filter_func = filter['f']
filter_val = filter['f.v']
if filter_func(message, filter_val):
break
else:
marker['next'] = message_id
yield utils.basic_message(message, now)
yield _it(message_ids, filters)
yield marker['next']
def get_metadata(self, name, project):
if project is None:
project = ''
try:
sel = sa.sql.select([tables.Queues.c.metadata],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name))
return json.loads(self.driver.get(sel)[0])
except utils.NoResult:
raise errors.QueueDoesNotExist(name, project)
def bulk_delete(self, queue, message_ids, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
pipe = self._pipeline
for message_id in message_ids:
pipe.delete(message_id).zrem(messages_set_id, message_id)
pipe.execute()
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = {'c': 1, 'cl': 0, 'm': metadata, 't': timeutils.utcnow_ts()}
if self.exists(name, project):
q_info['c'] = self._get_queue_info(q_id, 'c')
q_info['cl'] = self._get_queue_info(q_id, 'cl')
self._client.hmset(q_id, q_info)
def get_metadata(self, name, project):
if project is None:
project = ''
try:
return self.driver.get(
'''
select metadata from Queues
where project = ? and name = ?''', project, name)[0]
except utils.NoResult:
raise errors.QueueDoesNotExist(name, project)
def set_metadata(self, name, metadata, project):
if project is None:
project = ''
self.driver.run(
'''
update Queues
set metadata = ?
where project = ? and name = ?
''', self.driver.pack(metadata), project, name)
if not self.driver.affected:
raise errors.QueueDoesNotExist(name, project)
def delete(self, name, project=None):
# Note(prashanthr_): Pipelining is used to ensure no race conditions
# occur.
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
pipe.zrem(qset_id, q_id)
pipe.delete(q_id)
self.driver.message_controller._delete_queue_messages(name, project)
pipe.execute()
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_name = utils.scope_queue_name(name, project)
project_bkt = self._get_project_bkt(project)
q_info = project_bkt.get(q_name)
q_data = q_info.data
# Set the new metadata.
q_data['m'] = metadata
q_data['t'] = timeutils.utcnow_ts()
# Store the back into the bucket.
# Note(prashanthr_): Check if re-indexing is needed.
q_info.add_index(QUEUE_MODTIME_INDEX, q_data['t'])
q_info.store()
def set_metadata(self, name, metadata, project):
if project is None:
project = ''
update = tables.Queues.update().\
where(sa.and_(
tables.Queues.c.project == project,
tables.Queues.c.name == name)).\
values(metadata=json.dumps(metadata))
res = self.driver.run(update)
try:
if res.rowcount != 1:
raise errors.QueueDoesNotExist(name, project)
finally:
res.close()
def bulk_get(self, queue, message_ids, project=None):
# Note(prashanthr_): Pipeline is not used here as atomic guarantee
# is not required, but can be used for performance.
client = self._client
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
def _it(message_ids):
now = timeutils.utcnow_ts()
for message_id in message_ids:
message = client.hgetall(message_id)
if message:
yield utils.basic_message(message, now)
return _it(message_ids)
def post(self, queue, messages, client_uuid, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
message_ids = []
num_messages = self._get_count(messages_set_id)
prepared_messages = [{
'id':
utils.generate_uuid(),
't':
message['ttl'],
'e':
now_dt + datetime.timedelta(seconds=message['ttl']),
'u':
client_uuid,
'c':
None,
'k':
num_messages + index,
'c.e':
now,
'b':
message['body'] if 'body' in message else {},
} for index, message in enumerate(messages)]
pipe = self._pipeline
self._queue_controller._inc_counter(queue, project, len(messages))
for message in prepared_messages:
m_id = message['id']
message_ids.append(m_id)
pipe.zadd(messages_set_id, message['c.e'], m_id)
pipe.hmset(m_id, message)
pipe.execute()
return message_ids
def delete(self, queue, message_id, project=None, claim=None):
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
pipe = self._pipeline
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
if not self._exists(queue, project, message_id):
raise errors.MessageDoesNotExist(message_id, queue, project)
now = timeutils.utcnow_ts()
message = self._get(message_id)
msg_claim = message['c']
is_claimed = (msg_claim is not 'None' and int(message['c.e']) > now)
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
else:
if msg_claim != claim:
raise errors.MessageIsClaimedBy(message_id, msg_claim)
pipe.delete(message_id).zrem(messages_set_id, message_id).execute()
def get(self, queue, message_id, project):
target = self._lookup(queue, project)
if target:
control = target.message_controller
return control.get(queue, message_id=message_id, project=project)
raise errors.QueueDoesNotExist(project, queue)
def stats(self, name, project=None):
target = self._lookup(name, project)
if target:
control = target.queue_controller
return control.stats(name, project=project)
raise errors.QueueDoesNotExist(name, project)
def set_metadata(self, name, metadata, project=None):
key = self._queue(project, name)
if not self._db.exists(key):
raise errors.QueueDoesNotExist(name, project)
self._db.hset(key, 'm', msgpack.dumps(metadata))
