def count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue
'p': project,
'q': queue_name,
# The messages can not be expired
'e': {'$gt': timeutils.utcnow_ts()},
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
return self._col.find(query).hint(COUNTING_INDEX_FIELDS).count()
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow_ts()},
}
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
preference = pymongo.read_preferences.ReadPreference.PRIMARY
collection = self._collection(queue_name, project)
msgs = collection.find(query, sort=[('k', 1)],
read_preference=preference).hint(
CLAIMED_INDEX_FIELDS
)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
'p_q': utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow_ts()},
}
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
preference = pymongo.read_preferences.ReadPreference.PRIMARY
collection = self._collection(queue_name, project)
msgs = collection.find(query, sort=[('k', 1)],
read_preference=preference)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
controller = self.driver.message_controller
active = controller._count(name,
project=project,
include_claimed=False)
total = controller._count(name, project=project, include_claimed=True)
message_stats = {
'claimed': total - active,
'free': active,
'total': total,
}
try:
oldest = controller.first(name, project=project, sort=1)
newest = controller.first(name, project=project, sort=-1)
except errors.QueueIsEmpty:
pass
else:
now = timeutils.utcnow_ts()
message_stats['oldest'] = utils.stat_message(oldest, now)
message_stats['newest'] = utils.stat_message(newest, now)
return {'messages': message_stats}
def _count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
collection = self._collection(queue_name, project)
return collection.find(query).hint(COUNTING_INDEX_FIELDS).count()
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = self._client.hgetall(q_id)
claimed = int(q_info['cl'])
total = int(q_info['c'])
msg_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts()
message_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total
}
try:
newest = msg_ctrl.first(name, project, 1)
oldest = msg_ctrl.first(name, project, -1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = utils.stat_message(newest, now)
message_stats['oldest'] = utils.stat_message(oldest, now)
return {'messages': message_stats}
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = self._client.hgetall(q_id)
claimed = int(q_info['cl'])
total = int(q_info['c'])
msg_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts()
message_stats = {
'claimed': claimed,
'free': total-claimed,
'total': total
}
try:
newest = msg_ctrl.first(name, project, 1)
oldest = msg_ctrl.first(name, project, -1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = utils.stat_message(newest, now)
message_stats['oldest'] = utils.stat_message(oldest, now)
return {'messages': message_stats}
def _exists_unlocked(self, key):
now = timeutils.utcnow_ts()
try:
timeout = self._cache[key][0]
return not timeout or now <= timeout
except KeyError:
return False
def claimed(self, queue, claim_id=None, expires=None, limit=None):
query = {
"c.id": claim_id,
"q": utils.to_oid(queue),
}
if not claim_id:
# lookup over c.id to use the index
query["c.id"] = {"$ne": None}
if expires:
query["c.e"] = {"$gt": expires}
msgs = self._col.find(query, sort=[("_id", 1)])
if limit:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
oid = msg.get("_id")
age = now - utils.oid_ts(oid)
return {
"id": str(oid),
"age": age,
"ttl": msg["t"],
"body": msg["b"],
"claim": msg["c"]
}
return utils.HookedCursor(msgs, denormalizer)
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update({
PROJ_QUEUE: scope,
'c.id': cid
}, {'$set': {
'c': {
'id': None,
'e': now
}
}},
upsert=False,
multi=True)
def bulk_get(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if not message_ids:
return iter([])
now = timeutils.utcnow_ts()
# Base query, always check expire time
query = {
'_id': {
'$in': message_ids
},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
# NOTE(flaper87): Should this query
# be sorted?
messages = collection.find(query).hint(ID_INDEX_FIELDS)
def denormalizer(msg):
return _basic_message(msg, now)
return utils.HookedCursor(messages, denormalizer)
def list(self, queue_name, project=None, marker=None, limit=None,
echo=False, client_uuid=None, include_claimed=False):
if limit is None:
limit = self.driver.limits_conf.default_message_paging
if marker is not None:
try:
marker = int(marker)
except ValueError:
yield iter([])
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed, limit=limit)
marker_id = {}
now = timeutils.utcnow_ts()
# NOTE (kgriffs) @utils.raises_conn_error not needed on this
# function, since utils.HookedCursor already has it.
def denormalizer(msg):
marker_id['next'] = msg['k']
return _basic_message(msg, now)
yield utils.HookedCursor(messages, denormalizer)
yield str(marker_id['next'])
def _it(message_ids):
now = timeutils.utcnow_ts()
for message_id in message_ids:
message = client.hgetall(message_id)
if message:
yield utils.basic_message(message, now)
def get(self, queue_name, message_id, project=None):
"""Gets a single message by ID.
:raises: exceptions.MessageDoesNotExist
"""
mid = utils.to_oid(message_id)
if mid is None:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
now = timeutils.utcnow_ts()
query = {
'_id': mid,
'p': project,
'q': queue_name,
'e': {'$gt': now}
}
message = list(self._col.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
return _basic_message(message[0], now)
def _count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
collection = self._collection(queue_name, project)
return collection.find(query).hint(COUNTING_INDEX_FIELDS).count()
def _get_unlocked(self, key, default=None):
now = timeutils.utcnow_ts()
try:
timeout, value = self._cache[key]
except KeyError:
return (0, default)
if timeout and now >= timeout:
# NOTE(flaper87): Record expired,
# remove it from the cache but catch
# KeyError and ValueError in case
# _purge_expired removed this key already.
try:
del self._cache[key]
except KeyError:
pass
try:
# NOTE(flaper87): Keys with ttl == 0
# don't exist in the _keys_expires dict
self._keys_expires[timeout].remove(key)
except (KeyError, ValueError):
pass
return (0, default)
return (timeout, value)
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
controller = self.driver.message_controller
active = controller._count(name, project=project,
include_claimed=False)
total = controller._count(name, project=project,
include_claimed=True)
message_stats = {
'claimed': total - active,
'free': active,
'total': total,
}
try:
oldest = controller.first(name, project=project, sort=1)
newest = controller.first(name, project=project, sort=-1)
except errors.QueueIsEmpty:
pass
else:
now = timeutils.utcnow_ts()
message_stats['oldest'] = utils.stat_message(oldest, now)
message_stats['newest'] = utils.stat_message(newest, now)
return {'messages': message_stats}
def list(self, queue_name, project=None, marker=None, limit=None,
echo=False, client_uuid=None, include_claimed=False):
if limit is None:
limit = CFG.default_message_paging
if marker is not None:
try:
marker = int(marker)
except ValueError:
yield iter([])
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed, limit=limit)
marker_id = {}
now = timeutils.utcnow_ts()
def denormalizer(msg):
marker_id['next'] = msg['k']
return _basic_message(msg, now)
yield utils.HookedCursor(messages, denormalizer)
yield str(marker_id['next'])
def list(self, queue_name, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None, include_claimed=False):
if marker is not None:
try:
marker = int(marker)
except ValueError:
yield iter([])
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed, limit=limit)
marker_id = {}
now = timeutils.utcnow_ts()
# NOTE (kgriffs) @utils.raises_conn_error not needed on this
# function, since utils.HookedCursor already has it.
def denormalizer(msg):
marker_id['next'] = msg['k']
return _basic_message(msg, now)
yield utils.HookedCursor(messages, denormalizer)
yield str(marker_id['next'])
def _remove_expired(self, queue_name, project):
"""Removes all expired messages except for the most recent
in each queue.
This method is used in lieu of mongo's TTL index since we
must always leave at least one message in the queue for
calculating the next marker.
:param queue_name: name for the queue from which to remove
expired messages
:param project: Project queue_name belong's too
"""
# Get the message with the highest marker, and leave
# it in the queue
head = self._col.find_one({'p': project, 'q': queue_name},
sort=[('k', -1)], fields={'k': 1})
if head is None:
# Assume queue was just deleted via a parallel request
LOG.debug(_(u'Queue %s is empty or missing.') % queue_name)
return
query = {
'p': project,
'q': queue_name,
'k': {'$ne': head['k']},
'e': {'$lte': timeutils.utcnow_ts()},
}
self._col.remove(query, w=0)
def list(self,
project=None,
marker=None,
limit=storage.DEFAULT_QUEUES_PER_PAGE,
detailed=False):
start = marker or None
limit = limit if limit else storage.DEFAULT_MESSAGES_PER_CLAIM
project_bkt = self._get_project_bkt(project)
now = timeutils.utcnow_ts()
results_obj = project_bkt.get_index(QUEUE_MODTIME_INDEX,
0,
continuation=start,
max_results=limit,
endkey=now)
result_keys = results_obj.results
def _it():
for q_name in result_keys:
queue = project_bkt.get(q_name).data
yield _normalize(queue, q_name, detailed)
yield _it()
yield results_obj.continuation
def bulk_get(self, queue, message_ids, project):
if project is None:
project = ''
message_ids = [id for id in
map(utils.msgid_decode, message_ids)
if id is not None]
statement = sa.sql.select([tables.Messages.c.id,
tables.Messages.c.body,
tables.Messages.c.ttl,
tables.Messages.c.created])
and_stmt = [tables.Messages.c.id.in_(message_ids),
tables.Queues.c.name == queue,
tables.Queues.c.project == project,
tables.Messages.c.ttl >
sfunc.now() - tables.Messages.c.created]
j = sa.join(tables.Messages, tables.Queues,
tables.Messages.c.qid == tables.Queues.c.id)
statement = statement.select_from(j).where(sa.and_(*and_stmt))
now = timeutils.utcnow_ts()
records = self.driver.run(statement)
for id, body, ttl, created in records:
yield {
'id': utils.msgid_encode(id),
'ttl': ttl,
'age': now - calendar.timegm(created.timetuple()),
'body': json.loads(body),
}
def _set_claim_counter(self, q_id, count):
q_info = {
'c': self._get_queue_info(q_id, 'c'),
'cl': count,
'm': self._get_queue_info(q_id, 'm'),
't': timeutils.utcnow_ts()
}
self._client.hmset(q_id, q_info)
def _set_claim_counter(self, q_id, count):
q_info = {
'c': self._get_queue_info(q_id, 'c'),
'cl': count,
'm': self._get_queue_info(q_id, 'm'),
't': timeutils.utcnow_ts()
}
self._client.hmset(q_id, q_info)
def _inc_counter(self, name, project=None, amount=1, window=None):
"""Increments the message counter and returns the new value.
:param name: Name of the queue to which the counter is scoped
:param project: Queue's project name
:param amount: (Default 1) Amount by which to increment the counter
:param window: (Default None) A time window, in seconds, that
must have elapsed since the counter was last updated, in
order to increment the counter.
:returns: Updated message counter value, or None if window
was specified, and the counter has already been updated
within the specified time period.
:raises: storage.errors.QueueDoesNotExist
"""
now = timeutils.utcnow_ts()
update = {'$inc': {'c.v': amount}, '$set': {'c.t': now}}
query = _get_scoped_query(name, project)
if window is not None:
threshold = now - window
query['c.t'] = {'$lt': threshold}
while True:
try:
doc = self._collection.find_and_modify(query,
update,
new=True,
fields={
'c.v': 1,
'_id': 0
})
break
except pymongo.errors.AutoReconnect as ex:
LOG.exception(ex)
if doc is None:
if window is None:
# NOTE(kgriffs): Since we did not filter by a time window,
# the queue should have been found and updated. Perhaps
# the queue has been deleted?
message = _(u'Failed to increment the message '
u'counter for queue %(name)s and '
u'project %(project)s')
message %= dict(name=name, project=project)
LOG.warning(message)
raise errors.QueueDoesNotExist(name, project)
# NOTE(kgriffs): Assume the queue existed, but the counter
# was recently updated, causing the range query on 'c.t' to
# exclude the record.
return None
return doc['c']['v']
def first(self, queue_name, project=None, sort=1):
cursor = self._list(queue_name, project=project, include_claimed=True, sort=sort, limit=1)
try:
message = next(cursor)
except StopIteration:
raise errors.QueueIsEmpty(queue_name, project)
now = timeutils.utcnow_ts()
return _basic_message(message, now)
def get(self, queue, message_id, project):
body, ttl, created = self._get(queue, message_id, project)
now = timeutils.utcnow_ts()
return {
'id': message_id,
'ttl': ttl,
'age': now - calendar.timegm(created.timetuple()),
'body': json.loads(body),
}
def get(self, queue, message_id, project):
body, ttl, created = self._get(queue, message_id, project)
now = timeutils.utcnow_ts()
return {
'id': message_id,
'ttl': ttl,
'age': now - calendar.timegm(created.timetuple()),
'body': json.loads(body),
}
def get(self, queue, message_id, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
if not self._exists(queue, project, message_id):
raise errors.MessageDoesNotExist(message_id, queue, project)
msg = self._get(message_id)
return utils.basic_message(msg, timeutils.utcnow_ts())
def _set_unlocked(self, key, value, ttl=0):
expires_at = 0
if ttl != 0:
expires_at = timeutils.utcnow_ts() + ttl
self._cache[key] = (expires_at, value)
if expires_at:
self._keys_expires[expires_at].add(key)
def list(self,
queue,
project=None,
marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=True,
client_uuid=None,
include_claimed=False):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
client = self._client
start = client.zrank(messages_set_id, marker) or 0
# In a pooled environment, the default values are not being set.
limit = limit or storage.DEFAULT_MESSAGES_PER_CLAIM
message_ids = client.zrange(messages_set_id, start, start + limit)
filters = collections.defaultdict(dict)
# Build a list of filters for checking the following:
# 1. Message is claimed.
# 2. echo message to the client.
if not include_claimed:
filters['claimed_filter']['f'] = utils.msg_claimed_filter
filters['claimed_filter']['f.v'] = timeutils.utcnow_ts()
if not echo:
filters['echo_filter']['f'] = utils.msg_echo_filter
filters['echo_filter']['f.v'] = client_uuid
marker = {}
def _it(message_ids, filters={}):
"""Create a filtered iterator of messages.
The function accepts a list of filters to be filtered
before the the message can be included as a part of the reply.
"""
now = timeutils.utcnow_ts()
for message_id in message_ids:
message = client.hgetall(message_id)
if message:
for filter in six.itervalues(filters):
filter_func = filter['f']
filter_val = filter['f.v']
if filter_func(message, filter_val):
break
else:
marker['next'] = message_id
yield utils.basic_message(message, now)
yield _it(message_ids, filters)
yield marker['next']
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
ttl = int(metadata.get('ttl', 60))
expires = now + ttl
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update({
'p_q': scope,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False,
multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update({
'p_q': scope,
'e': {
'$lt': expires
},
'c.id': cid
}, {'$set': {
'e': expires,
't': ttl
}},
upsert=False,
multi=True)
def it():
now = timeutils.utcnow_ts()
for id, body, ttl, created in records:
marker_id['next'] = id
yield {
'id': utils.msgid_encode(id),
'ttl': ttl,
'age': now - calendar.timegm(created.timetuple()),
'body': json.loads(body),
}
def _incr_append(self, key, other):
with lockutils.lock(key):
timeout, value = self._get_unlocked(key)
if value is None:
return None
ttl = timeutils.utcnow_ts() - timeout
new_value = value + other
self._set_unlocked(key, new_value, ttl)
return new_value
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = {'c': 1, 'cl': 0, 'm': metadata, 't': timeutils.utcnow_ts()}
if self.exists(name, project):
q_info['c'] = self._get_queue_info(q_id, 'c')
q_info['cl'] = self._get_queue_info(q_id, 'cl')
self._client.hmset(q_id, q_info)
def pop(self, queue_name, limit, project=None):
if project is None:
project = ''
with self.driver.trans() as trans:
sel = sa.sql.select([
tables.Messages.c.id, tables.Messages.c.body,
tables.Messages.c.ttl, tables.Messages.c.created
])
j = sa.join(tables.Messages, tables.Queues,
tables.Messages.c.qid == tables.Queues.c.id)
sel = sel.select_from(j)
and_clause = [
tables.Queues.c.name == queue_name,
tables.Queues.c.project == project
]
and_clause.append(tables.Messages.c.cid == (None))
sel = sel.where(sa.and_(*and_clause))
sel = sel.limit(limit)
records = trans.execute(sel)
now = timeutils.utcnow_ts()
messages = []
message_ids = []
for id, body, ttl, created in records:
messages.append({
'id':
utils.msgid_encode(id),
'ttl':
ttl,
'age':
now - calendar.timegm(created.timetuple()),
'body':
utils.json_decode(body),
})
message_ids.append(id)
statement = tables.Messages.delete()
qid = utils.get_qid(self.driver, queue_name, project)
and_stmt = [
tables.Messages.c.id.in_(message_ids),
tables.Messages.c.qid == qid
]
trans.execute(statement.where(sa.and_(*and_stmt)))
return messages
def _inc_counter(self, name, project=None, amount=1, window=None):
"""Increments the message counter and returns the new value.
:param name: Name of the queue to which the counter is scoped
:param project: Queue's project name
:param amount: (Default 1) Amount by which to increment the counter
:param window: (Default None) A time window, in seconds, that
must have elapsed since the counter was last updated, in
order to increment the counter.
:returns: Updated message counter value, or None if window
was specified, and the counter has already been updated
within the specified time period.
:raises: storage.errors.QueueDoesNotExist
"""
now = timeutils.utcnow_ts()
update = {'$inc': {'c.v': amount}, '$set': {'c.t': now}}
query = _get_scoped_query(name, project)
if window is not None:
threshold = now - window
query['c.t'] = {'$lt': threshold}
while True:
try:
doc = self._collection.find_and_modify(
query, update, new=True, fields={'c.v': 1, '_id': 0})
break
except pymongo.errors.AutoReconnect as ex:
LOG.exception(ex)
if doc is None:
if window is None:
# NOTE(kgriffs): Since we did not filter by a time window,
# the queue should have been found and updated. Perhaps
# the queue has been deleted?
message = _(u'Failed to increment the message '
u'counter for queue %(name)s and '
u'project %(project)s')
message %= dict(name=name, project=project)
LOG.warning(message)
raise errors.QueueDoesNotExist(name, project)
# NOTE(kgriffs): Assume the queue existed, but the counter
# was recently updated, causing the range query on 'c.t' to
# exclude the record.
return None
return doc['c']['v']
def set(self, key, value, ttl=0):
key = self._prepare_key(key)
with lockutils.lock(key):
expires_at = 0
if ttl != 0:
expires_at = timeutils.utcnow_ts() + ttl
self._cache[key] = (expires_at, value)
if expires_at:
self._keys_expires.setdefault(expires_at, set()).add(key)
return True
def first(self, queue_name, project=None, sort=1):
cursor = self._list(queue_name,
project=project,
include_claimed=True,
sort=sort,
limit=1)
try:
message = next(cursor)
except StopIteration:
raise errors.QueueIsEmpty(queue_name, project)
now = timeutils.utcnow_ts()
return _basic_message(message, now)
def unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
self._col.update({'p': project, 'q': queue_name, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}},
upsert=False, multi=True)
def test_msg_claimed_filter(self):
now = timeutils.utcnow_ts()
msg = {'c': 'None', 'c.e': now}
self.assertFalse(utils.msg_claimed_filter(msg, now))
msg = {'c': utils.generate_uuid(), 'c.e': now + 60}
self.assertTrue(utils.msg_claimed_filter(msg, now))
msg['c.e'] = now - 60
self.assertFalse(utils.msg_claimed_filter(msg, now))
def create(self, name, project=None):
q_name = utils.scope_queue_name(name, project)
project_bkt = self._get_project_bkt(project)
if self.exists(name, project):
return False
q_info = {'c': 1, 'm': {}, 't': timeutils.utcnow_ts()}
# Riak python client supports automatic
# serialization of py dicts <-> JSON.
q_obj = project_bkt.new(q_name, q_info)
q_obj.add_index(QUEUE_MODTIME_INDEX, q_info['t'])
q_obj.store()
return True
def get(self, queue, message_id, tenant=None):
# Base query, always check expire time
mid = utils.to_oid(message_id)
query = {
"q": self._get_queue_id(queue, tenant),
"e": {"$gt": timeutils.utcnow_ts()},
"_id": mid
}
message = self._col.find_one(query)
if message is None:
raise exceptions.MessageDoesNotExist(mid, queue, tenant)
oid = message.get("_id")
age = timeutils.utcnow_ts() - utils.oid_ts(oid)
return {
"id": oid,
"age": age,
"ttl": message["t"],
"body": message["b"],
}
def get(self, key, default=None):
key = self._prepare_key(key)
with lockutils.lock(key):
now = timeutils.utcnow_ts()
try:
timeout, value = self._cache[key]
if timeout and now >= timeout:
del self._cache[key]
return default
return value
except KeyError:
return default
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
# NOTE(cpp-cabrera): return early - the user gaves us an
# invalid claim id and that renders the rest of this
# request moot
cid = utils.to_oid(claim)
if cid is None:
return
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
is_claimed = (message['c']['id'] is not None and
message['c']['e'] > now)
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
pref = pymongo.read_preferences.ReadPreference.PRIMARY
message = collection.find_one(query, read_preference=pref)
if message['c']['id'] != cid:
raise errors.MessageIsClaimedBy(message_id, claim)
collection.remove(query['_id'], w=0)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
# NOTE(cpp-cabrera): return early - the user gaves us an
# invalid claim id and that renders the rest of this
# request moot
cid = utils.to_oid(claim)
if cid is None:
return
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
is_claimed = (message['c']['id'] is not None
and message['c']['e'] > now)
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
pref = pymongo.read_preferences.ReadPreference.PRIMARY
message = collection.find_one(query, read_preference=pref)
if message['c']['id'] != cid:
raise errors.MessageIsClaimedBy(message_id, claim)
collection.remove(query['_id'], w=0)
def get(self, queue_name, message_id, project=None):
mid = utils.to_oid(message_id)
if mid is None:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
now = timeutils.utcnow_ts()
query = {"_id": mid, PROJ_QUEUE: utils.scope_queue_name(queue_name, project)}
collection = self._collection(queue_name, project)
message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
return _basic_message(message[0], now)
def create(self, name, project=None):
# Note(prashanthr_): Implement as a lua script.
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
# Check if the queue already exists.
if self._client.zrank(qset_id, q_id) is not None:
return False
# Pipeline ensures atomic inserts.
q_info = {'c': 1, 'cl': 0, 'm': {}, 't': timeutils.utcnow_ts()}
pipe.zadd(qset_id, 1, q_id).hmset(q_id, q_info)
return all(map(bool, pipe.execute()))
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_name = utils.scope_queue_name(name, project)
project_bkt = self._get_project_bkt(project)
q_info = project_bkt.get(q_name)
q_data = q_info.data
# Set the new metadata.
q_data['m'] = metadata
q_data['t'] = timeutils.utcnow_ts()
# Store the back into the bucket.
# Note(prashanthr_): Check if re-indexing is needed.
q_info.add_index(QUEUE_MODTIME_INDEX, q_data['t'])
q_info.store()
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update(
{PROJ_QUEUE: scope, "c.id": cid}, {"$set": {"c": {"id": None, "e": now}}}, upsert=False, multi=True
)
def test_basic_message(self):
msg_id = uuid.uuid4().hex
now = timeutils.utcnow_ts()
msg = {
'id': msg_id,
'c.e': now - 5,
't': '300',
'b': 'Hello Earthlings',
}
basic_msg = utils.basic_message(msg, now)
self.assertEqual(basic_msg['id'], msg_id)
self.assertEqual(basic_msg['age'], 5)
self.assertEqual(basic_msg['body'], 'Hello Earthlings')
self.assertEqual(basic_msg['ttl'], '300')
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = {
'c': 1,
'cl': 0,
'm': metadata,
't': timeutils.utcnow_ts()
}
if self.exists(name, project):
q_info['c'] = self._get_queue_info(q_id, 'c')
q_info['cl'] = self._get_queue_info(q_id, 'cl')
self._client.hmset(q_id, q_info)
