def test_scope_queue_name(self):
self.assertEqual(utils.scope_queue_name('my-q'), '/my-q')
self.assertEqual(utils.scope_queue_name('my-q', None), '/my-q')
self.assertEqual(utils.scope_queue_name('my-q', '123'), '123/my-q')
self.assertEqual(utils.scope_queue_name(None), '/')
self.assertEqual(utils.scope_queue_name(None, '123'), '123/')
def _count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
collection = self._collection(queue_name, project)
return collection.find(query).hint(COUNTING_INDEX_FIELDS).count()
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(claim_expires +
grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update_many({
'p_q': scope,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update_many(
{
'p_q': scope,
'e': {
'$lt': claim_expires_dt
},
'c.id': cid
}, {'$set': {
'e': message_expires,
't': message_ttl
}},
upsert=False)
def bulk_get(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if not message_ids:
return iter([])
now = timeutils.utcnow_ts()
# Base query, always check expire time
query = {
'_id': {
'$in': message_ids
},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
# NOTE(flaper87): Should this query
# be sorted?
messages = collection.find(query).hint(ID_INDEX_FIELDS)
def denormalizer(msg):
return _basic_message(msg, now)
return utils.HookedCursor(messages, denormalizer)
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update({
PROJ_QUEUE: scope,
'c.id': cid
}, {'$set': {
'c': {
'id': None,
'e': now
}
}},
upsert=False,
multi=True)
def _create(self, name, metadata=None, project=None):
# NOTE(flaper87): If the connection fails after it was called
# and we retry to insert the queue, we could end up returning
# `False` because of the `DuplicatedKeyError` although the
# queue was indeed created by this API call.
#
# TODO(kgriffs): Commented out `retries_on_autoreconnect` for
# now due to the above issue, since creating a queue is less
# important to make super HA.
try:
# NOTE(kgriffs): Start counting at 1, and assume the first
# message ever posted will succeed and set t to a UNIX
# "modified at" timestamp.
counter = {'v': 1, 't': 0}
scoped_name = utils.scope_queue_name(name, project)
self._collection.insert({
'p_q': scoped_name,
'm': metadata or {},
'c': counter
})
except pymongo.errors.DuplicateKeyError:
return False
else:
return True
def bulk_delete(self,
queue_name,
message_ids,
project=None,
claim_ids=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if claim_ids:
claim_ids = [cid for cid in map(utils.to_oid, claim_ids) if cid]
query = {
'_id': {
'$in': message_ids
},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
if claim_ids:
message_claim_ids = []
messages = collection.find(query).hint(ID_INDEX_FIELDS)
for message in messages:
message_claim_ids.append(message['c']['id'])
for cid in claim_ids:
if cid not in message_claim_ids:
raise errors.ClaimDoesNotExist(cid, queue_name, project)
collection.delete_many(query)
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow_ts()},
}
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
preference = pymongo.read_preferences.ReadPreference.PRIMARY
collection = self._collection(queue_name, project)
msgs = collection.find(query, sort=[('k', 1)],
read_preference=preference).hint(
CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def _count(self, topic_name, project=None, include_claimed=False):
"""Return total number of messages in a topic.
This method is designed to very quickly count the number
of messages in a given topic. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_TOPIC: utils.scope_queue_name(topic_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
collection = self._collection(topic_name, project)
return collection.count(filter=query, hint=COUNTING_INDEX_FIELDS)
def _create(self, name, metadata=None, project=None):
# NOTE(flaper87): If the connection fails after it was called
# and we retry to insert the queue, we could end up returning
# `False` because of the `DuplicatedKeyError` although the
# queue was indeed created by this API call.
#
# TODO(kgriffs): Commented out `retries_on_autoreconnect` for
# now due to the above issue, since creating a queue is less
# important to make super HA.
try:
# NOTE(kgriffs): Start counting at 1, and assume the first
# message ever posted will succeed and set t to a UNIX
# "modified at" timestamp.
counter = {'v': 1, 't': 0}
scoped_name = utils.scope_queue_name(name, project)
self._collection.insert_one(
{'p_q': scoped_name, 'm': metadata or {},
'c': counter})
except pymongo.errors.DuplicateKeyError:
return False
else:
return True
def _count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
collection = self._collection(queue_name, project)
return collection.count(filter=query, hint=COUNTING_INDEX_FIELDS)
def create(self, queue, subscriber, ttl, options, project=None):
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
source_query = {'p_q': utils.scope_queue_name(source, project)}
target_source = self._queue_collection.find_one(source_query,
fields={
'm': 1,
'_id': 0
})
if target_source is None:
raise errors.QueueDoesNotExist(target_source, project)
try:
subscription_id = self._collection.insert({
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project
})
return subscription_id
except pymongo.errors.DuplicateKeyError:
return None
def bulk_delete(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
query = {
'_id': {'$in': message_ids},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
collection.remove(query, w=0)
def bulk_delete(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
query = {
'_id': {'$in': message_ids},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
collection.remove(query, w=0)
def get(self, project, queue):
fields = {'_id': 0}
key = utils.scope_queue_name(queue, project)
entry = self._col.find_one({PRIMARY_KEY: key}, projection=fields)
if entry is None:
raise errors.QueueNotMapped(queue, project)
return _normalize(entry)
def get(self, project, queue):
fields = {'_id': 0}
key = utils.scope_queue_name(queue, project)
entry = self._col.find_one({PRIMARY_KEY: key},
projection=fields)
if entry is None:
raise errors.QueueNotMapped(queue, project)
return _normalize(entry)
def post(self, queue_name, messages, client_uuid, project=None):
# NOTE(flaper87): This method should be safe to retry on
# autoreconnect, since we've a 2-step insert for messages.
# The worst-case scenario is that we'll increase the counter
# several times and we'd end up with some non-active messages.
if not self._queue_ctrl.exists(queue_name, project):
raise errors.QueueDoesNotExist(queue_name, project)
# NOTE(flaper87): Make sure the counter exists. This method
# is an upsert.
self._get_counter(queue_name, project)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
collection = self._collection(queue_name, project)
messages = list(messages)
msgs_n = len(messages)
next_marker = self._inc_counter(queue_name, project,
amount=msgs_n) - msgs_n
prepared_messages = [{
PROJ_QUEUE:
utils.scope_queue_name(queue_name, project),
't':
message['ttl'],
'e':
now_dt + datetime.timedelta(seconds=message['ttl']),
'u':
client_uuid,
'c': {
'id': None,
'e': now,
'c': 0
},
'd':
now + message.get('delay', 0),
'b':
message['body'] if 'body' in message else {},
'k':
next_marker + index,
'tx':
None,
} for index, message in enumerate(messages)]
res = collection.insert_many(prepared_messages,
bypass_document_validation=True)
return [str(id_) for id_ in res.inserted_ids]
def _purge_queue(self, queue_name, project=None):
"""Removes all messages from the queue.
Warning: Only use this when deleting the queue; otherwise
you can cause a side-effect of reseting the marker counter
which can cause clients to miss tons of messages.
If the queue does not exist, this method fails silently.
:param queue_name: name of the queue to purge
:param project: ID of the project to which the queue belongs
"""
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.remove({PROJ_QUEUE: scope}, w=0)
def _purge_queue(self, queue_name, project=None):
"""Removes all messages from the queue.
Warning: Only use this when deleting the queue; otherwise
you can cause a side-effect of reseting the marker counter
which can cause clients to miss tons of messages.
If the queue does not exist, this method fails silently.
:param queue_name: name of the queue to purge
:param project: ID of the project to which the queue belongs
"""
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.delete_many({PROJ_QUEUE: scope})
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(claim, queue_name, project)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
kwargs = {}
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
message = collection.find_one(query, **kwargs)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.delete_one(query)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(claim, queue_name, project)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
kwargs = {}
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
message = collection.find_one(query, **kwargs)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.remove(query['_id'], w=0)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(
claim_expires + grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue, cid, expires=now,
limit=1, project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update({'p_q': scope, 'c.id': cid},
{'$set': {'c': meta}},
upsert=False, multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update({'p_q': scope,
'e': {'$lt': claim_expires_dt},
'c.id': cid},
{'$set': {'e': message_expires,
't': message_ttl}},
upsert=False, multi=True)
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update_many({PROJ_QUEUE: scope, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}},
upsert=False)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(queue_name, project, claim)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
pref = pymongo.read_preferences.ReadPreference.PRIMARY
message = collection.find_one(query, read_preference=pref)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.remove(query['_id'], w=0)
def post(self, queue_name, messages, client_uuid, project=None):
# NOTE(flaper87): This method should be safe to retry on
# autoreconnect, since we've a 2-step insert for messages.
# The worst-case scenario is that we'll increase the counter
# several times and we'd end up with some non-active messages.
if not self._queue_ctrl.exists(queue_name, project):
raise errors.QueueDoesNotExist(queue_name, project)
# NOTE(flaper87): Make sure the counter exists. This method
# is an upsert.
self._get_counter(queue_name, project)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
collection = self._collection(queue_name, project)
messages = list(messages)
msgs_n = len(messages)
next_marker = self._inc_counter(queue_name,
project,
amount=msgs_n) - msgs_n
prepared_messages = []
for index, message in enumerate(messages):
msg = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
't': message['ttl'],
'e': now_dt + datetime.timedelta(seconds=message['ttl']),
'u': client_uuid,
'c': {'id': None, 'e': now, 'c': 0},
'd': now + message.get('delay', 0),
'b': message['body'] if 'body' in message else {},
'k': next_marker + index,
'tx': None
}
if self.driver.conf.enable_checksum:
msg['cs'] = s_utils.get_checksum(message.get('body', None))
prepared_messages.append(msg)
res = collection.insert_many(prepared_messages,
bypass_document_validation=True)
return [str(id_) for id_ in res.inserted_ids]
def get(self, queue_name, message_id, project=None):
mid = utils.to_oid(message_id)
if mid is None:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
now = timeutils.utcnow_ts()
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
return _basic_message(message[0], now)
def _claimed(self,
queue_name,
claim_id,
expires=None,
limit=None,
project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {
'$gt': expires or timeutils.utcnow_ts()
},
}
kwargs = {}
collection = self._collection(queue_name, project)
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
msgs = collection.find(query, sort=[('k', 1)],
**kwargs).hint(CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def get(self, queue_name, message_id, project=None):
mid = utils.to_oid(message_id)
if mid is None:
raise errors.MessageDoesNotExist(message_id, queue_name,
project)
now = timeutils.utcnow_ts()
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise errors.MessageDoesNotExist(message_id, queue_name,
project)
return _basic_message(message[0], now)
def create(self, queue, subscriber, ttl, options, project=None):
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
source_query = {'p_q': utils.scope_queue_name(source, project)}
target_source = self._queue_collection.find_one(source_query,
fields={'m': 1,
'_id': 0})
if target_source is None:
raise errors.QueueDoesNotExist(target_source, project)
try:
subscription_id = self._collection.insert({'s': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project})
return subscription_id
except pymongo.errors.DuplicateKeyError:
return None
def bulk_delete(self, queue_name, message_ids, project=None,
claim_ids=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if claim_ids:
claim_ids = [cid for cid in map(utils.to_oid, claim_ids) if cid]
query = {
'_id': {'$in': message_ids},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
if claim_ids:
message_claim_ids = []
messages = collection.find(query).hint(ID_INDEX_FIELDS)
for message in messages:
message_claim_ids.append(message['c']['id'])
for cid in claim_ids:
if cid not in message_claim_ids:
raise errors.ClaimDoesNotExist(cid, queue_name, project)
collection.delete_many(query)
def pop(self, topic_name, limit, project=None):
query = {
PROJ_TOPIC: utils.scope_queue_name(topic_name, project),
}
# Only include messages that are not part of
# any claim, or are part of an expired claim.
now = timeutils.utcnow_ts()
query['c.e'] = {'$lte': now}
collection = self._collection(topic_name, project)
projection = {'_id': 1, 't': 1, 'b': 1, 'c.id': 1}
messages = (collection.find_one_and_delete(query,
projection=projection)
for _ in range(limit))
final_messages = [
_basic_message(message, now) for message in messages if message
]
return final_messages
def pop(self, queue_name, limit, project=None):
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
# Only include messages that are not part of
# any claim, or are part of an expired claim.
now = timeutils.utcnow_ts()
query['c.e'] = {'$lte': now}
collection = self._collection(queue_name, project)
projection = {'_id': 1, 't': 1, 'b': 1, 'c.id': 1}
messages = (collection.find_one_and_delete(query,
projection=projection)
for _ in range(limit))
final_messages = [_basic_message(message, now)
for message in messages
if message]
return final_messages
def bulk_get(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if not message_ids:
return iter([])
now = timeutils.utcnow_ts()
# Base query, always check expire time
query = {
'_id': {'$in': message_ids},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
# NOTE(flaper87): Should this query
# be sorted?
messages = collection.find(query).hint(ID_INDEX_FIELDS)
def denormalizer(msg):
return _basic_message(msg, now)
return utils.HookedCursor(messages, denormalizer)
def pop(self, queue_name, limit, project=None):
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
# Only include messages that are not part of
# any claim, or are part of an expired claim.
now = timeutils.utcnow_ts()
query['c.e'] = {'$lte': now}
collection = self._collection(queue_name, project)
fields = {'_id': 1, 't': 1, 'b': 1, 'c.id': 1}
messages = (collection.find_and_modify(query,
fields=fields,
remove=True)
for _ in range(limit))
final_messages = [
_basic_message(message, now) for message in messages if message
]
return final_messages
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow_ts()},
}
kwargs = {}
collection = self._collection(queue_name, project)
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
msgs = collection.find(query, sort=[('k', 1)], **kwargs).hint(
CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def create(self,
queue,
metadata,
project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
"""Creates a claim.
This implementation was done in a best-effort fashion.
In order to create a claim we need to get a list
of messages that can be claimed. Once we have that
list we execute a query filtering by the ids returned
by the previous query.
Since there's a lot of space for race conditions here,
we'll check if the number of updated records is equal to
the max number of messages to claim. If the number of updated
messages is lower than limit we'll try to claim the remaining
number of messages.
This 2 queries are required because there's no way, as for the
time being, to execute an update on a limited number of records.
"""
msg_ctrl = self.driver.message_controller
queue_ctrl = self.driver.queue_controller
# Get the maxClaimCount, deadLetterQueue and DelayTTL
# from current queue's meta
queue_meta = queue_ctrl.get(queue, project=project)
ttl = metadata['ttl']
grace = metadata['grace']
oid = objectid.ObjectId()
now = timeutils.utcnow_ts()
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expiration = datetime.datetime.utcfromtimestamp(claim_expires +
grace)
meta = {
'id': oid,
't': ttl,
'e': claim_expires,
'c': 0 # NOTE(flwang): A placeholder which will be updated later
}
# NOTE(cdyangzhenyu): If the ``_default_message_delay`` is 0 means
# queue is not delayed queue, So we don't filter for delay messages.
include_delayed = False if queue_meta.get('_default_message_delay',
0) else True
# Get a list of active, not claimed nor expired
# messages that could be claimed.
msgs = msg_ctrl._active(queue,
projection={
'_id': 1,
'c': 1
},
project=project,
limit=limit,
include_delayed=include_delayed)
messages = iter([])
be_claimed = [(msg['_id'], msg['c'].get('c', 0)) for msg in msgs]
ids = [_id for _id, _ in be_claimed]
if len(ids) == 0:
return None, messages
# NOTE(kgriffs): Set the claim field for
# the active message batch, while also
# filtering out any messages that happened
# to get claimed just now by one or more
# parallel requests.
#
# Filtering by just 'c.e' works because
# new messages have that field initialized
# to the current time when the message is
# posted. There is no need to check whether
# 'c' exists or 'c.id' is None.
collection = msg_ctrl._collection(queue, project)
updated = collection.update_many(
{
'_id': {
'$in': ids
},
'c.e': {
'$lte': now
}
}, {'$set': {
'c': meta
}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
new_values = {'e': message_expiration, 't': message_ttl}
collection.update_many(
{
'p_q': utils.scope_queue_name(queue, project),
'e': {
'$lt': claim_expires_dt
},
'c.id': oid
}, {'$set': new_values},
upsert=False)
msg_count_moved_to_DLQ = 0
if ('_max_claim_count' in queue_meta
and '_dead_letter_queue' in queue_meta):
LOG.debug(u"The list of messages being claimed: %(be_claimed)s",
{"be_claimed": be_claimed})
for _id, claimed_count in be_claimed:
# NOTE(flwang): We have claimed the message above, but we will
# update the claim count below. So that means, when the
# claimed_count equals queue_meta['_max_claim_count'], the
# message has met the threshold. And Zaqar will move it to the
# DLQ.
if claimed_count < queue_meta['_max_claim_count']:
# 1. Save the new max claim count for message
collection.update_one({
'_id': _id,
'c.id': oid
}, {'$set': {
'c.c': claimed_count + 1
}},
upsert=False)
LOG.debug(
u"Message %(id)s has been claimed %(count)d "
u"times.", {
"id": str(_id),
"count": claimed_count + 1
})
else:
# 2. Check if the message's claim count has exceeded the
# max claim count defined in the queue, if so, move the
# message to the dead letter queue.
# NOTE(flwang): We're moving message directly. That means,
# the queue and dead letter queue must be created on the
# same storage pool. It's a technical tradeoff, because if
# we re-send the message to the dead letter queue by
# message controller, then we will lost all the claim
# information.
dlq_name = queue_meta['_dead_letter_queue']
new_msg = {
'c.c': claimed_count,
'p_q': utils.scope_queue_name(dlq_name, project)
}
dlq_ttl = queue_meta.get("_dead_letter_queue_messages_ttl")
if dlq_ttl:
new_msg['t'] = dlq_ttl
kwargs = {"return_document": ReturnDocument.AFTER}
msg = collection.find_one_and_update(
{
'_id': _id,
'c.id': oid
}, {'$set': new_msg}, **kwargs)
dlq_collection = msg_ctrl._collection(dlq_name, project)
if not dlq_collection:
LOG.warning(
u"Failed to find the message collection "
u"for queue %(dlq_name)s", {"dlq_name": dlq_name})
return None, iter([])
# NOTE(flwang): If dead letter queue and queue are in the
# same partition, the message has been already
# modified.
if collection != dlq_collection:
result = dlq_collection.insert_one(msg)
if result.inserted_id:
collection.delete_one({'_id': _id})
LOG.debug(
u"Message %(id)s has met the max claim count "
u"%(count)d, now it has been moved to dead "
u"letter queue %(dlq_name)s.", {
"id": str(_id),
"count": claimed_count,
"dlq_name": dlq_name
})
msg_count_moved_to_DLQ += 1
if updated.modified_count != 0:
# NOTE(kgriffs): This extra step is necessary because
# in between having gotten a list of active messages
# and updating them, some of them may have been
# claimed by a parallel request. Therefore, we need
# to find out which messages were actually tagged
# with the claim ID successfully.
if msg_count_moved_to_DLQ < updated.modified_count:
claim, messages = self.get(queue, oid, project=project)
else:
# NOTE(flwang): Though messages are claimed, but all of them
# have met the max claim count and have been moved to DLQ.
return None, iter([])
return str(oid), messages
def create(self,
queue,
metadata,
project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
"""Creates a claim.
This implementation was done in a best-effort fashion.
In order to create a claim we need to get a list
of messages that can be claimed. Once we have that
list we execute a query filtering by the ids returned
by the previous query.
Since there's a lot of space for race conditions here,
we'll check if the number of updated records is equal to
the max number of messages to claim. If the number of updated
messages is lower than limit we'll try to claim the remaining
number of messages.
This 2 queries are required because there's no way, as for the
time being, to execute an update on a limited number of records.
"""
msg_ctrl = self.driver.message_controller
ttl = metadata['ttl']
grace = metadata['grace']
oid = objectid.ObjectId()
now = timeutils.utcnow_ts()
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expiration = datetime.datetime.utcfromtimestamp(claim_expires +
grace)
meta = {
'id': oid,
't': ttl,
'e': claim_expires,
}
# Get a list of active, not claimed nor expired
# messages that could be claimed.
msgs = msg_ctrl._active(queue,
fields={'_id': 1},
project=project,
limit=limit)
messages = iter([])
ids = [msg['_id'] for msg in msgs]
if len(ids) == 0:
return (None, messages)
now = timeutils.utcnow_ts()
# NOTE(kgriffs): Set the claim field for
# the active message batch, while also
# filtering out any messages that happened
# to get claimed just now by one or more
# parallel requests.
#
# Filtering by just 'c.e' works because
# new messages have that field initialized
# to the current time when the message is
# posted. There is no need to check whether
# 'c' exists or 'c.id' is None.
collection = msg_ctrl._collection(queue, project)
updated = collection.update({
'_id': {
'$in': ids
},
'c.e': {
'$lte': now
}
}, {'$set': {
'c': meta
}},
upsert=False,
multi=True)['n']
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
new_values = {'e': message_expiration, 't': message_ttl}
collection.update(
{
'p_q': utils.scope_queue_name(queue, project),
'e': {
'$lt': claim_expires_dt
},
'c.id': oid
}, {'$set': new_values},
upsert=False,
multi=True)
if updated != 0:
# NOTE(kgriffs): This extra step is necessary because
# in between having gotten a list of active messages
# and updating them, some of them may have been
# claimed by a parallel request. Therefore, we need
# to find out which messages were actually tagged
# with the claim ID successfully.
claim, messages = self.get(queue, oid, project=project)
return (str(oid), messages)
def _get_scoped_query(name, project):
return {'p_t': utils.scope_queue_name(name, project)}
def _list(self,
topic_name,
project=None,
marker=None,
echo=False,
client_uuid=None,
projection=None,
include_claimed=False,
include_delayed=False,
sort=1,
limit=None):
"""Message document listing helper.
:param topic_name: Name of the topic to list
:param project: (Default None) Project `topic_name` belongs to. If
not specified, queries the "global" namespace/project.
:param marker: (Default None) Message marker from which to start
iterating. If not specified, starts with the first message
available in the topic.
:param echo: (Default False) Whether to return messages that match
client_uuid
:param client_uuid: (Default None) UUID for the client that
originated this request
:param projection: (Default None) a list of field names that should be
returned in the result set or a dict specifying the fields to
include or exclude
:param include_claimed: (Default False) Whether to include
claimed messages, not just active ones
:param include_delayed: (Default False) Whether to include
delayed messages, not just active ones
:param sort: (Default 1) Sort order for the listing. Pass 1 for
ascending (oldest message first), or -1 for descending (newest
message first).
:param limit: (Default None) The maximum number of messages
to list. The results may include fewer messages than the
requested `limit` if not enough are available. If limit is
not specified
:returns: Generator yielding up to `limit` messages.
"""
if sort not in (1, -1):
raise ValueError(u'sort must be either 1 (ascending) '
u'or -1 (descending)')
now = timeutils.utcnow_ts()
query = {
# Messages must belong to this topic and project.
PROJ_TOPIC: utils.scope_queue_name(topic_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not echo:
query['u'] = {'$ne': client_uuid}
if marker is not None:
query['k'] = {'$gt': marker}
collection = self._collection(topic_name, project)
if not include_delayed:
# NOTE(cdyangzhenyu): Only include messages that are not
# part of any delay, or are part of an expired delay. if
# the message has no attribute 'd', it will also be obtained.
# This is for compatibility with old data.
query['$or'] = [{'d': {'$lte': now}}, {'d': {'$exists': False}}]
# Construct the request
cursor = collection.find(query,
projection=projection,
sort=[('k', sort)])
if limit is not None:
cursor.limit(limit)
# NOTE(flaper87): Suggest the index to use for this query to
# ensure the most performant one is chosen.
return cursor.hint(ACTIVE_INDEX_FIELDS)
def delete(self, project, queue):
self._col.remove({PRIMARY_KEY: utils.scope_queue_name(queue, project)},
w=0)
def exists(self, project, queue):
key = utils.scope_queue_name(queue, project)
return self._col.find_one({PRIMARY_KEY: key}) is not None
def _insert(self, project, queue, pool, upsert):
key = utils.scope_queue_name(queue, project)
return self._col.update({PRIMARY_KEY: key},
{'$set': {'s': pool}}, upsert=upsert)
def post(self, queue_name, messages, client_uuid, project=None):
# NOTE(flaper87): This method should be safe to retry on
# autoreconnect, since we've a 2-step insert for messages.
# The worst-case scenario is that we'll increase the counter
# several times and we'd end up with some non-active messages.
if not self._queue_ctrl.exists(queue_name, project):
raise errors.QueueDoesNotExist(queue_name, project)
# NOTE(flaper87): Make sure the counter exists. This method
# is an upsert.
self._get_counter(queue_name, project)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
collection = self._collection(queue_name, project)
# Set the next basis marker for the first attempt.
#
# Note that we don't increment the counter right away because
# if 2 concurrent posts happen and the one with the higher counter
# ends before the one with the lower counter, there's a window
# where a client paging through the queue may get the messages
# with the higher counter and skip the previous ones. This would
# make our FIFO guarantee unsound.
next_marker = self._get_counter(queue_name, project)
# Unique transaction ID to facilitate atomic batch inserts
transaction = objectid.ObjectId()
prepared_messages = []
for index, message in enumerate(messages):
msg = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
't': message['ttl'],
'e': now_dt + datetime.timedelta(seconds=message['ttl']),
'u': client_uuid,
'c': {'id': None, 'e': now, 'c': 0},
'd': now + message.get('delay', 0),
'b': message['body'] if 'body' in message else {},
'k': next_marker + index,
'tx': None
}
if self.driver.conf.enable_checksum:
msg['cs'] = s_utils.get_checksum(message.get('body', None))
prepared_messages.append(msg)
# NOTE(kgriffs): Don't take the time to do a 2-phase insert
# if there is no way for it to partially succeed.
if len(prepared_messages) == 1:
transaction = None
prepared_messages[0]['tx'] = None
# Use a retry range for sanity, although we expect
# to rarely, if ever, reach the maximum number of
# retries.
#
# NOTE(kgriffs): With the default configuration (100 ms
# max sleep, 1000 max attempts), the max stall time
# before the operation is abandoned is 49.95 seconds.
for attempt in self._retry_range:
try:
res = collection.insert_many(prepared_messages,
bypass_document_validation=True)
# Log a message if we retried, for debugging perf issues
if attempt != 0:
msgtmpl = _(u'%(attempts)d attempt(s) required to post '
u'%(num_messages)d messages to queue '
u'"%(queue)s" under project %(project)s')
LOG.debug(msgtmpl,
dict(queue=queue_name,
attempts=attempt + 1,
num_messages=len(res.inserted_ids),
project=project))
# Update the counter in preparation for the next batch
#
# NOTE(kgriffs): Due to the unique index on the messages
# collection, competing inserts will fail as a whole,
# and keep retrying until the counter is incremented
# such that the competing marker's will start at a
# unique number, 1 past the max of the messages just
# inserted above.
self._inc_counter(queue_name, project,
amount=len(res.inserted_ids))
# NOTE(kgriffs): Finalize the insert once we can say that
# all the messages made it. This makes bulk inserts
# atomic, assuming queries filter out any non-finalized
# messages.
if transaction is not None:
collection.update_many({'tx': transaction},
{'$set': {'tx': None}},
upsert=False)
return [str(id_) for id_ in res.inserted_ids]
except (pymongo.errors.DuplicateKeyError,
pymongo.errors.BulkWriteError) as ex:
# TODO(kgriffs): Record stats of how often retries happen,
# and how many attempts, on average, are required to insert
# messages.
# NOTE(kgriffs): This can be used in conjunction with the
# log line, above, that is emitted after all messages have
# been posted, to gauge how long it is taking for messages
# to be posted to a given queue, or overall.
#
# TODO(kgriffs): Add transaction ID to help match up loglines
if attempt == 0:
msgtmpl = _(u'First attempt failed while '
u'adding messages to queue '
u'"%(queue)s" under project %(project)s')
LOG.debug(msgtmpl, dict(queue=queue_name, project=project))
# NOTE(kgriffs): Never retry past the point that competing
# messages expire and are GC'd, since once they are gone,
# the unique index no longer protects us from getting out
# of order, which could cause an observer to miss this
# message. The code below provides a sanity-check to ensure
# this situation can not happen.
elapsed = timeutils.utcnow_ts() - now
if elapsed > MAX_RETRY_POST_DURATION:
msgtmpl = (u'Exceeded maximum retry duration for queue '
u'"%(queue)s" under project %(project)s')
LOG.warning(msgtmpl,
dict(queue=queue_name, project=project))
break
# Chill out for a moment to mitigate thrashing/thundering
self._backoff_sleep(attempt)
# NOTE(kgriffs): Perhaps we failed because a worker crashed
# after inserting messages, but before incrementing the
# counter; that would cause all future requests to stall,
# since they would keep getting the same base marker that is
# conflicting with existing messages, until the messages that
# "won" expire, at which time we would end up reusing markers,
# and that could make some messages invisible to an observer
# that is querying with a marker that is large than the ones
# being reused.
#
# To mitigate this, we apply a heuristic to determine whether
# a counter has stalled. We attempt to increment the counter,
# but only if it hasn't been updated for a few seconds, which
# should mean that nobody is left to update it!
#
# Note that we increment one at a time until the logjam is
# broken, since we don't know how many messages were posted
# by the worker before it crashed.
next_marker = self._inc_counter(
queue_name, project, window=COUNTER_STALL_WINDOW)
# Retry the entire batch with a new sequence of markers.
#
# NOTE(kgriffs): Due to the unique index, and how
# MongoDB works with batch requests, we will never
# end up with a partially-successful update. The first
# document in the batch will fail to insert, and the
# remainder of the documents will not be attempted.
if next_marker is None:
# NOTE(kgriffs): Usually we will end up here, since
# it should be rare that a counter becomes stalled.
next_marker = self._get_counter(
queue_name, project)
else:
msgtmpl = (u'Detected a stalled message counter '
u'for queue "%(queue)s" under '
u'project %(project)s.'
u'The counter was incremented to %(value)d.')
LOG.warning(msgtmpl,
dict(queue=queue_name,
project=project,
value=next_marker))
for index, message in enumerate(prepared_messages):
message['k'] = next_marker + index
except bsonerror.InvalidDocument as ex:
LOG.exception(ex)
raise
except Exception as ex:
LOG.exception(ex)
raise
msgtmpl = (u'Hit maximum number of attempts (%(max)s) for queue '
u'"%(queue)s" under project %(project)s')
LOG.warning(msgtmpl,
dict(max=self.driver.mongodb_conf.max_attempts,
queue=queue_name,
project=project))
raise errors.MessageConflict(queue_name, project)
def _list(self, queue_name, project=None, marker=None,
echo=False, client_uuid=None, projection=None,
include_claimed=False, include_delayed=False,
sort=1, limit=None):
"""Message document listing helper.
:param queue_name: Name of the queue to list
:param project: (Default None) Project `queue_name` belongs to. If
not specified, queries the "global" namespace/project.
:param marker: (Default None) Message marker from which to start
iterating. If not specified, starts with the first message
available in the queue.
:param echo: (Default False) Whether to return messages that match
client_uuid
:param client_uuid: (Default None) UUID for the client that
originated this request
:param projection: (Default None) a list of field names that should be
returned in the result set or a dict specifying the fields to
include or exclude
:param include_claimed: (Default False) Whether to include
claimed messages, not just active ones
:param include_delayed: (Default False) Whether to include
delayed messages, not just active ones
:param sort: (Default 1) Sort order for the listing. Pass 1 for
ascending (oldest message first), or -1 for descending (newest
message first).
:param limit: (Default None) The maximum number of messages
to list. The results may include fewer messages than the
requested `limit` if not enough are available. If limit is
not specified
:returns: Generator yielding up to `limit` messages.
"""
if sort not in (1, -1):
raise ValueError(u'sort must be either 1 (ascending) '
u'or -1 (descending)')
now = timeutils.utcnow_ts()
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not echo:
query['u'] = {'$ne': client_uuid}
if marker is not None:
query['k'] = {'$gt': marker}
collection = self._collection(queue_name, project)
if not include_claimed:
# Only include messages that are not part of
# any claim, or are part of an expired claim.
query['c.e'] = {'$lte': now}
if not include_delayed:
# NOTE(cdyangzhenyu): Only include messages that are not
# part of any delay, or are part of an expired delay. if
# the message has no attribute 'd', it will also be obtained.
# This is for compatibility with old data.
query['$or'] = [{'d': {'$lte': now}},
{'d': {'$exists': False}}]
# Construct the request
cursor = collection.find(query,
projection=projection,
sort=[('k', sort)])
if limit is not None:
cursor.limit(limit)
# NOTE(flaper87): Suggest the index to use for this query to
# ensure the most performant one is chosen.
return cursor.hint(ACTIVE_INDEX_FIELDS)
def create(self, queue, metadata, project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
"""Creates a claim.
This implementation was done in a best-effort fashion.
In order to create a claim we need to get a list
of messages that can be claimed. Once we have that
list we execute a query filtering by the ids returned
by the previous query.
Since there's a lot of space for race conditions here,
we'll check if the number of updated records is equal to
the max number of messages to claim. If the number of updated
messages is lower than limit we'll try to claim the remaining
number of messages.
This 2 queries are required because there's no way, as for the
time being, to execute an update on a limited number of records.
"""
msg_ctrl = self.driver.message_controller
ttl = metadata['ttl']
grace = metadata['grace']
oid = objectid.ObjectId()
now = timeutils.utcnow_ts()
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expiration = datetime.datetime.utcfromtimestamp(
claim_expires + grace)
meta = {
'id': oid,
't': ttl,
'e': claim_expires,
}
# Get a list of active, not claimed nor expired
# messages that could be claimed.
msgs = msg_ctrl._active(queue, projection={'_id': 1}, project=project,
limit=limit)
messages = iter([])
ids = [msg['_id'] for msg in msgs]
if len(ids) == 0:
return None, messages
now = timeutils.utcnow_ts()
# NOTE(kgriffs): Set the claim field for
# the active message batch, while also
# filtering out any messages that happened
# to get claimed just now by one or more
# parallel requests.
#
# Filtering by just 'c.e' works because
# new messages have that field initialized
# to the current time when the message is
# posted. There is no need to check whether
# 'c' exists or 'c.id' is None.
collection = msg_ctrl._collection(queue, project)
updated = collection.update({'_id': {'$in': ids},
'c.e': {'$lte': now}},
{'$set': {'c': meta}},
upsert=False,
multi=True)['n']
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
new_values = {'e': message_expiration, 't': message_ttl}
collection.update({'p_q': utils.scope_queue_name(queue, project),
'e': {'$lt': claim_expires_dt},
'c.id': oid},
{'$set': new_values},
upsert=False, multi=True)
if updated != 0:
# NOTE(kgriffs): This extra step is necessary because
# in between having gotten a list of active messages
# and updating them, some of them may have been
# claimed by a parallel request. Therefore, we need
# to find out which messages were actually tagged
# with the claim ID successfully.
claim, messages = self.get(queue, oid, project=project)
return str(oid), messages
def create(self, queue, metadata, project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
"""Creates a claim.
This implementation was done in a best-effort fashion.
In order to create a claim we need to get a list
of messages that can be claimed. Once we have that
list we execute a query filtering by the ids returned
by the previous query.
Since there's a lot of space for race conditions here,
we'll check if the number of updated records is equal to
the max number of messages to claim. If the number of updated
messages is lower than limit we'll try to claim the remaining
number of messages.
This 2 queries are required because there's no way, as for the
time being, to execute an update on a limited number of records.
"""
msg_ctrl = self.driver.message_controller
queue_ctrl = self.driver.queue_controller
# Get the maxClaimCount, deadLetterQueue and DelayTTL
# from current queue's meta
queue_meta = queue_ctrl.get(queue, project=project)
ttl = metadata['ttl']
grace = metadata['grace']
oid = objectid.ObjectId()
now = timeutils.utcnow_ts()
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expiration = datetime.datetime.utcfromtimestamp(
claim_expires + grace)
meta = {
'id': oid,
't': ttl,
'e': claim_expires,
'c': 0 # NOTE(flwang): A placeholder which will be updated later
}
# NOTE(cdyangzhenyu): If the ``_default_message_delay`` is 0 means
# queue is not delayed queue, So we don't filter for delay messages.
include_delayed = False if queue_meta.get('_default_message_delay',
0) else True
# Get a list of active, not claimed nor expired
# messages that could be claimed.
msgs = msg_ctrl._active(queue, projection={'_id': 1, 'c': 1},
project=project,
limit=limit,
include_delayed=include_delayed)
messages = iter([])
be_claimed = [(msg['_id'], msg['c'].get('c', 0)) for msg in msgs]
ids = [_id for _id, _ in be_claimed]
if len(ids) == 0:
return None, messages
# NOTE(kgriffs): Set the claim field for
# the active message batch, while also
# filtering out any messages that happened
# to get claimed just now by one or more
# parallel requests.
#
# Filtering by just 'c.e' works because
# new messages have that field initialized
# to the current time when the message is
# posted. There is no need to check whether
# 'c' exists or 'c.id' is None.
collection = msg_ctrl._collection(queue, project)
updated = collection.update_many({'_id': {'$in': ids},
'c.e': {'$lte': now}},
{'$set': {'c': meta}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
new_values = {'e': message_expiration, 't': message_ttl}
collection.update_many({'p_q': utils.scope_queue_name(queue, project),
'e': {'$lt': claim_expires_dt},
'c.id': oid},
{'$set': new_values},
upsert=False)
msg_count_moved_to_DLQ = 0
if ('_max_claim_count' in queue_meta and
'_dead_letter_queue' in queue_meta):
LOG.debug(u"The list of messages being claimed: %(be_claimed)s",
{"be_claimed": be_claimed})
for _id, claimed_count in be_claimed:
# NOTE(flwang): We have claimed the message above, but we will
# update the claim count below. So that means, when the
# claimed_count equals queue_meta['_max_claim_count'], the
# message has met the threshold. And Zaqar will move it to the
# DLQ.
if claimed_count < queue_meta['_max_claim_count']:
# 1. Save the new max claim count for message
collection.update_one({'_id': _id,
'c.id': oid},
{'$set': {'c.c': claimed_count + 1}},
upsert=False)
LOG.debug(u"Message %(id)s has been claimed %(count)d "
u"times.", {"id": str(_id),
"count": claimed_count + 1})
else:
# 2. Check if the message's claim count has exceeded the
# max claim count defined in the queue, if so, move the
# message to the dead letter queue.
# NOTE(flwang): We're moving message directly. That means,
# the queue and dead letter queue must be created on the
# same storage pool. It's a technical tradeoff, because if
# we re-send the message to the dead letter queue by
# message controller, then we will lost all the claim
# information.
dlq_name = queue_meta['_dead_letter_queue']
new_msg = {'c.c': claimed_count,
'p_q': utils.scope_queue_name(dlq_name,
project)}
dlq_ttl = queue_meta.get("_dead_letter_queue_messages_ttl")
if dlq_ttl:
new_msg['t'] = dlq_ttl
kwargs = {"return_document": ReturnDocument.AFTER}
msg = collection.find_one_and_update({'_id': _id,
'c.id': oid},
{'$set': new_msg},
**kwargs)
dlq_collection = msg_ctrl._collection(dlq_name, project)
if not dlq_collection:
LOG.warning(u"Failed to find the message collection "
u"for queue %(dlq_name)s", {"dlq_name":
dlq_name})
return None, iter([])
# NOTE(flwang): If dead letter queue and queue are in the
# same partition, the message has been already
# modified.
if collection != dlq_collection:
result = dlq_collection.insert_one(msg)
if result.inserted_id:
collection.delete_one({'_id': _id})
LOG.debug(u"Message %(id)s has met the max claim count "
u"%(count)d, now it has been moved to dead "
u"letter queue %(dlq_name)s.",
{"id": str(_id), "count": claimed_count,
"dlq_name": dlq_name})
msg_count_moved_to_DLQ += 1
if updated.modified_count != 0:
# NOTE(kgriffs): This extra step is necessary because
# in between having gotten a list of active messages
# and updating them, some of them may have been
# claimed by a parallel request. Therefore, we need
# to find out which messages were actually tagged
# with the claim ID successfully.
if msg_count_moved_to_DLQ < updated.modified_count:
claim, messages = self.get(queue, oid, project=project)
else:
# NOTE(flwang): Though messages are claimed, but all of them
# have met the max claim count and have been moved to DLQ.
return None, iter([])
return str(oid), messages
def _insert(self, project, queue, pool, upsert):
key = utils.scope_queue_name(queue, project)
return self._col.update({PRIMARY_KEY: key}, {'$set': {
's': pool
}},
upsert=upsert)
def _list(self,
queue_name,
project=None,
marker=None,
echo=False,
client_uuid=None,
projection=None,
include_claimed=False,
sort=1,
limit=None):
"""Message document listing helper.
:param queue_name: Name of the queue to list
:param project: (Default None) Project `queue_name` belongs to. If
not specified, queries the "global" namespace/project.
:param marker: (Default None) Message marker from which to start
iterating. If not specified, starts with the first message
available in the queue.
:param echo: (Default False) Whether to return messages that match
client_uuid
:param client_uuid: (Default None) UUID for the client that
originated this request
:param fields: (Default None) Fields to include in emitted
documents
:param include_claimed: (Default False) Whether to include
claimed messages, not just active ones
:param sort: (Default 1) Sort order for the listing. Pass 1 for
ascending (oldest message first), or -1 for descending (newest
message first).
:param limit: (Default None) The maximum number of messages
to list. The results may include fewer messages than the
requested `limit` if not enough are available. If limit is
not specified
:returns: Generator yielding up to `limit` messages.
"""
if sort not in (1, -1):
raise ValueError(u'sort must be either 1 (ascending) '
u'or -1 (descending)')
now = timeutils.utcnow_ts()
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not echo:
query['u'] = {'$ne': client_uuid}
if marker is not None:
query['k'] = {'$gt': marker}
collection = self._collection(queue_name, project)
if not include_claimed:
# Only include messages that are not part of
# any claim, or are part of an expired claim.
query['c.e'] = {'$lte': now}
# Construct the request
cursor = collection.find(query,
projection=projection,
sort=[('k', sort)])
if limit is not None:
cursor.limit(limit)
# NOTE(flaper87): Suggest the index to use for this query to
# ensure the most performant one is chosen.
return cursor.hint(ACTIVE_INDEX_FIELDS)
def delete(self, project, queue):
self._col.delete_one({
PRIMARY_KEY: utils.scope_queue_name(queue, project)})
def delete(self, project, queue):
self._col.remove({PRIMARY_KEY: utils.scope_queue_name(queue, project)},
w=0)
def exists(self, project, queue):
key = utils.scope_queue_name(queue, project)
return self._col.find_one({PRIMARY_KEY: key}) is not None
def _get_scoped_query(name, project):
return {'p_q': utils.scope_queue_name(name, project)}
def post(self, queue_name, messages, client_uuid, project=None):
# NOTE(flaper87): This method should be safe to retry on
# autoreconnect, since we've a 2-step insert for messages.
# The worst-case scenario is that we'll increase the counter
# several times and we'd end up with some non-active messages.
if not self._queue_ctrl.exists(queue_name, project):
raise errors.QueueDoesNotExist(queue_name, project)
# NOTE(flaper87): Make sure the counter exists. This method
# is an upsert.
self._get_counter(queue_name, project)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
collection = self._collection(queue_name, project)
# Set the next basis marker for the first attempt.
#
# Note that we don't increment the counter right away because
# if 2 concurrent posts happen and the one with the higher counter
# ends before the one with the lower counter, there's a window
# where a client paging through the queue may get the messages
# with the higher counter and skip the previous ones. This would
# make our FIFO guarantee unsound.
next_marker = self._get_counter(queue_name, project)
# Unique transaction ID to facilitate atomic batch inserts
transaction = objectid.ObjectId()
prepared_messages = [{
PROJ_QUEUE:
utils.scope_queue_name(queue_name, project),
't':
message['ttl'],
'e':
now_dt + datetime.timedelta(seconds=message['ttl']),
'u':
client_uuid,
'c': {
'id': None,
'e': now
},
'b':
message['body'] if 'body' in message else {},
'k':
next_marker + index,
'tx':
transaction,
} for index, message in enumerate(messages)]
# NOTE(kgriffs): Don't take the time to do a 2-phase insert
# if there is no way for it to partially succeed.
if len(prepared_messages) == 1:
transaction = None
prepared_messages[0]['tx'] = None
# Use a retry range for sanity, although we expect
# to rarely, if ever, reach the maximum number of
# retries.
#
# NOTE(kgriffs): With the default configuration (100 ms
# max sleep, 1000 max attempts), the max stall time
# before the operation is abandoned is 49.95 seconds.
for attempt in self._retry_range:
try:
ids = collection.insert(prepared_messages, check_keys=False)
# Log a message if we retried, for debugging perf issues
if attempt != 0:
msgtmpl = _(u'%(attempts)d attempt(s) required to post '
u'%(num_messages)d messages to queue '
u'"%(queue)s" under project %(project)s')
LOG.debug(
msgtmpl,
dict(queue=queue_name,
attempts=attempt + 1,
num_messages=len(ids),
project=project))
# Update the counter in preparation for the next batch
#
# NOTE(kgriffs): Due to the unique index on the messages
# collection, competing inserts will fail as a whole,
# and keep retrying until the counter is incremented
# such that the competing marker's will start at a
# unique number, 1 past the max of the messages just
# inserted above.
self._inc_counter(queue_name, project, amount=len(ids))
# NOTE(kgriffs): Finalize the insert once we can say that
# all the messages made it. This makes bulk inserts
# atomic, assuming queries filter out any non-finalized
# messages.
if transaction is not None:
collection.update({'tx': transaction},
{'$set': {
'tx': None
}},
upsert=False,
multi=True)
return [str(id_) for id_ in ids]
except pymongo.errors.DuplicateKeyError as ex:
# TODO(kgriffs): Record stats of how often retries happen,
# and how many attempts, on average, are required to insert
# messages.
# NOTE(kgriffs): This can be used in conjunction with the
# log line, above, that is emitted after all messages have
# been posted, to gauge how long it is taking for messages
# to be posted to a given queue, or overall.
#
# TODO(kgriffs): Add transaction ID to help match up loglines
if attempt == 0:
msgtmpl = _(u'First attempt failed while '
u'adding messages to queue '
u'"%(queue)s" under project %(project)s')
LOG.debug(msgtmpl, dict(queue=queue_name, project=project))
# NOTE(kgriffs): Never retry past the point that competing
# messages expire and are GC'd, since once they are gone,
# the unique index no longer protects us from getting out
# of order, which could cause an observer to miss this
# message. The code below provides a sanity-check to ensure
# this situation can not happen.
elapsed = timeutils.utcnow_ts() - now
if elapsed > MAX_RETRY_POST_DURATION:
msgtmpl = _LW(u'Exceeded maximum retry duration for queue '
u'"%(queue)s" under project %(project)s')
LOG.warning(msgtmpl, dict(queue=queue_name,
project=project))
break
# Chill out for a moment to mitigate thrashing/thundering
self._backoff_sleep(attempt)
# NOTE(kgriffs): Perhaps we failed because a worker crashed
# after inserting messages, but before incrementing the
# counter; that would cause all future requests to stall,
# since they would keep getting the same base marker that is
# conflicting with existing messages, until the messages that
# "won" expire, at which time we would end up reusing markers,
# and that could make some messages invisible to an observer
# that is querying with a marker that is large than the ones
# being reused.
#
# To mitigate this, we apply a heuristic to determine whether
# a counter has stalled. We attempt to increment the counter,
# but only if it hasn't been updated for a few seconds, which
# should mean that nobody is left to update it!
#
# Note that we increment one at a time until the logjam is
# broken, since we don't know how many messages were posted
# by the worker before it crashed.
next_marker = self._inc_counter(queue_name,
project,
window=COUNTER_STALL_WINDOW)
# Retry the entire batch with a new sequence of markers.
#
# NOTE(kgriffs): Due to the unique index, and how
# MongoDB works with batch requests, we will never
# end up with a partially-successful update. The first
# document in the batch will fail to insert, and the
# remainder of the documents will not be attempted.
if next_marker is None:
# NOTE(kgriffs): Usually we will end up here, since
# it should be rare that a counter becomes stalled.
next_marker = self._get_counter(queue_name, project)
else:
msgtmpl = _LW(u'Detected a stalled message counter '
u'for queue "%(queue)s" under '
u'project %(project)s.'
u'The counter was incremented to %(value)d.')
LOG.warning(
msgtmpl,
dict(queue=queue_name,
project=project,
value=next_marker))
for index, message in enumerate(prepared_messages):
message['k'] = next_marker + index
except Exception as ex:
LOG.exception(ex)
raise
msgtmpl = _LW(u'Hit maximum number of attempts (%(max)s) for queue '
u'"%(queue)s" under project %(project)s')
LOG.warning(
msgtmpl,
dict(max=self.driver.mongodb_conf.max_attempts,
queue=queue_name,
project=project))
raise errors.MessageConflict(queue_name, project)
