def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_SUBSCRIPTIONS_PER_PAGE):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': utils.to_oid(marker)}
projection = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1}
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
def normalizer(record):
ret = {
'id': str(record['_id']),
'source': record['s'],
'subscriber': record['u'],
'ttl': record['t'],
'options': record['o'],
}
marker_name['next'] = record['_id']
return ret
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def _list(self,
project=None,
marker=None,
limit=storage.DEFAULT_QUEUES_PER_PAGE,
detailed=False):
query = utils.scoped_query(marker, project)
projection = {'p_q': 1, '_id': 0}
if detailed:
projection['m'] = 1
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('p_q')
marker_name = {}
def normalizer(record):
queue = {'name': utils.descope_queue_name(record['p_q'])}
marker_name['next'] = queue['name']
if detailed:
queue['metadata'] = record['m']
return queue
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def _list(self,
project=None,
kfilter={},
marker=None,
limit=storage.DEFAULT_TOPICS_PER_PAGE,
detailed=False,
name=None):
query = utils.scoped_query(marker,
project,
name,
kfilter,
key_value='p_t')
projection = {'p_t': 1, '_id': 0}
if detailed:
projection['m'] = 1
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('p_t')
marker_name = {}
def normalizer(record):
topic = {'name': utils.descope_queue_name(record['p_t'])}
marker_name['next'] = topic['name']
if detailed:
topic['metadata'] = record['m']
return topic
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def _list_by_pool(self, pool, limit=10, detailed=False):
query = {'s': pool}
cursor = self._col.find(query, projection=_field_spec(detailed),
limit=limit).sort('n', 1)
normalizer = functools.partial(_normalize, detailed=detailed)
return utils.HookedCursor(cursor, normalizer)
def bulk_get(self, queue_name, message_ids, project=None):
message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
if not message_ids:
return iter([])
now = timeutils.utcnow_ts()
# Base query, always check expire time
query = {
'_id': {
'$in': message_ids
},
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
# NOTE(flaper87): Should this query
# be sorted?
messages = collection.find(query).hint(ID_INDEX_FIELDS)
def denormalizer(msg):
return _basic_message(msg, now)
return utils.HookedCursor(messages, denormalizer)
def list(self, queue, project=None, marker=None, limit=10):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': marker}
fields = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1}
cursor = self._collection.find(query, fields=fields)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
def normalizer(record):
ret = {
'id': str(record['_id']),
'source': record['s'],
'subscriber': record['u'],
'ttl': record['t'],
'options': record['o'],
}
marker_name['next'] = record['_id']
return ret
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def list(self, queue_name, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None, include_claimed=False):
if marker is not None:
try:
marker = int(marker)
except ValueError:
yield iter([])
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed, limit=limit)
marker_id = {}
now = timeutils.utcnow_ts()
# NOTE (kgriffs) @utils.raises_conn_error not needed on this
# function, since utils.HookedCursor already has it.
def denormalizer(msg):
marker_id['next'] = msg['k']
return _basic_message(msg, now)
yield utils.HookedCursor(messages, denormalizer)
yield str(marker_id['next'])
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow_ts()},
}
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
preference = pymongo.read_preferences.ReadPreference.PRIMARY
collection = self._collection(queue_name, project)
msgs = collection.find(query, sort=[('k', 1)],
read_preference=preference).hint(
CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def _get_pools_by_flavor(self, flavor=None, detailed=False):
query = None
if flavor is None:
query = {'f': None}
elif flavor.get('name') is not None:
query = {'f': flavor.get('name')}
cursor = self._col.find(query, projection=_field_spec(detailed))
normalizer = functools.partial(_normalize, detailed=detailed)
return utils.HookedCursor(cursor, normalizer)
def list(self, project=None, marker=None, limit=10, detailed=False):
query = {'p': project}
if marker is not None:
query['n'] = {'$gt': marker}
cursor = self._col.find(query, projection=_field_spec(detailed),
limit=limit).sort('n', 1)
marker_name = {}
def normalizer(flavor):
marker_name['next'] = flavor['n']
return _normalize(flavor, detailed=detailed)
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def list(self, marker=None, limit=10, detailed=False):
query = {}
if marker is not None:
query['n'] = {'$gt': marker}
cursor = self._col.find(query, fields=_field_spec(detailed),
limit=limit).sort('n')
marker_name = {}
def normalizer(pool):
marker_name['next'] = pool['n']
return _normalize(pool, detailed=detailed)
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def _claimed(self,
queue_name,
claim_id,
expires=None,
limit=None,
project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {
'$gt': expires or timeutils.utcnow_ts()
},
}
kwargs = {}
collection = self._collection(queue_name, project)
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
msgs = collection.find(query, sort=[('k', 1)],
**kwargs).hint(CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_SUBSCRIPTIONS_PER_PAGE):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': utils.to_oid(marker)}
projection = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1, 'c': 1}
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
now = timeutils.utcnow_ts()
def normalizer(record):
marker_name['next'] = record['_id']
return _basic_subscription(record, now)
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def list(self, project):
fields = {'_id': 0}
query = utils.scoped_query(None, project)
return utils.HookedCursor(self._col.find(query, fields), _normalize)
def _get_pools_by_group(self, group=None, detailed=False):
cursor = self._col.find({'g': group}, projection=_field_spec(detailed))
normalizer = functools.partial(_normalize, detailed=detailed)
return utils.HookedCursor(cursor, normalizer)
