def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(claim, queue_name, project)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
kwargs = {}
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
message = collection.find_one(query, **kwargs)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.remove(query['_id'], w=0)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(claim, queue_name, project)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
kwargs = {}
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
message = collection.find_one(query, **kwargs)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.delete_one(query)
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update({
PROJ_QUEUE: scope,
'c.id': cid
}, {'$set': {
'c': {
'id': None,
'e': now
}
}},
upsert=False,
multi=True)
def update(self, queue, subscription_id, project=None, **kwargs):
names = ('subscriber', 'ttl', 'options')
key_transform = lambda x: 'u' if x == 'subscriber' else x[0]
fields = common_utils.fields(kwargs,
names,
pred=lambda x: x is not None,
key_transform=key_transform)
assert fields, ('`subscriber`, `ttl`, '
'or `options` not found in kwargs')
new_ttl = fields.get('t', None)
if new_ttl is not None:
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
expires = now_dt + datetime.timedelta(seconds=new_ttl)
fields['e'] = expires
try:
res = self._collection.update(
{
'_id': utils.to_oid(subscription_id),
'p': project
}, {'$set': fields},
upsert=False)
except pymongo.errors.DuplicateKeyError:
raise errors.SubscriptionAlreadyExists()
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = _messages_iter(msg_ctrl._claimed(queue, cid, now,
project=project))
claim = next(msgs)
update_time = claim['e'] - claim['t']
age = now - update_time
claim_meta = {
'age': int(age),
'ttl': claim['t'],
'id': str(claim['id']),
}
except StopIteration:
raise errors.ClaimDoesNotExist(cid, queue, project)
return claim_meta, msgs
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(claim_expires +
grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update_many({
'p_q': scope,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update_many(
{
'p_q': scope,
'e': {
'$lt': claim_expires_dt
},
'c.id': cid
}, {'$set': {
'e': message_expires,
't': message_ttl
}},
upsert=False)
def confirm(self, queue, subscription_id, project=None, confirm=True):
res = self._collection.update({'_id': utils.to_oid(subscription_id),
'p': project}, {'$set': {'c': confirm}},
upsert=False)
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_SUBSCRIPTIONS_PER_PAGE):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': utils.to_oid(marker)}
projection = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1}
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
def normalizer(record):
ret = {
'id': str(record['_id']),
'source': record['s'],
'subscriber': record['u'],
'ttl': record['t'],
'options': record['o'],
}
marker_name['next'] = record['_id']
return ret
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = _messages_iter(
msg_ctrl._claimed(queue, cid, now, project=project))
claim = next(msgs)
update_time = claim['e'] - claim['t']
age = now - update_time
claim_meta = {
'age': int(age),
'ttl': claim['t'],
'id': str(claim['id']),
}
except StopIteration:
raise errors.ClaimDoesNotExist(cid, queue, project)
return claim_meta, msgs
def update(self, queue, subscription_id, project=None, **kwargs):
names = ('subscriber', 'ttl', 'options')
key_transform = lambda x: 'u' if x == 'subscriber' else x[0]
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None,
key_transform=key_transform)
assert fields, ('`subscriber`, `ttl`, '
'or `options` not found in kwargs')
new_ttl = fields.get('t', None)
if new_ttl is not None:
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
expires = now_dt + datetime.timedelta(seconds=new_ttl)
fields['e'] = expires
try:
res = self._collection.update(
{'_id': utils.to_oid(subscription_id),
'p': project},
{'$set': fields},
upsert=False)
except pymongo.errors.DuplicateKeyError:
raise errors.SubscriptionAlreadyExists()
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def get(self, queue, subscription_id, project=None):
res = self._collection.find_one({'_id': utils.to_oid(subscription_id),
'p': project})
if not res:
raise errors.SubscriptionDoesNotExist(subscription_id)
return _normalize(res)
def confirm(self, queue, subscription_id, project=None, confirmed=True):
res = self._collection.update({'_id': utils.to_oid(subscription_id),
'p': project},
{'$set': {'c': confirmed}},
upsert=False)
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(queue_name, project, claim)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
pref = pymongo.read_preferences.ReadPreference.PRIMARY
message = collection.find_one(query, read_preference=pref)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.remove(query['_id'], w=0)
def get(self, queue, subscription_id, project=None):
res = self._collection.find_one({'_id': utils.to_oid(subscription_id),
'p': project})
if not res:
raise errors.SubscriptionDoesNotExist(subscription_id)
now = timeutils.utcnow_ts()
return _basic_subscription(res, now)
def get(self, queue, subscription_id, project=None):
res = self._collection.find_one({'_id': utils.to_oid(subscription_id),
'p': project})
if not res:
raise errors.SubscriptionDoesNotExist(subscription_id)
now = timeutils.utcnow_ts()
return _basic_subscription(res, now)
def confirm(self, queue, subscription_id, project=None, confirmed=True):
res = self._collection.update_one(
{'_id': utils.to_oid(subscription_id),
'p': project},
{'$set': {'c': confirmed}},
upsert=False)
if res.matched_count == 0:
raise errors.SubscriptionDoesNotExist(subscription_id)
def update(self, queue, subscription_id, project=None, **kwargs):
names = ('subscriber', 'ttl', 'options')
key_transform = lambda x: 'u' if x == 'subscriber' else x[0]
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None,
key_transform=key_transform)
assert fields, ('`subscriber`, `ttl`, '
'or `options` not found in kwargs')
res = self._collection.update({'_id': utils.to_oid(subscription_id),
'p': project},
{'$set': fields},
upsert=False)
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(
claim_expires + grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue, cid, expires=now,
limit=1, project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update({'p_q': scope, 'c.id': cid},
{'$set': {'c': meta}},
upsert=False, multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update({'p_q': scope,
'e': {'$lt': claim_expires_dt},
'c.id': cid},
{'$set': {'e': message_expires,
't': message_ttl}},
upsert=False, multi=True)
def _unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow_ts()
scope = utils.scope_queue_name(queue_name, project)
collection = self._collection(queue_name, project)
collection.update_many({PROJ_QUEUE: scope, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}},
upsert=False)
def get(self, queue_name, message_id, project=None):
mid = utils.to_oid(message_id)
if mid is None:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
now = timeutils.utcnow_ts()
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise errors.MessageDoesNotExist(message_id, queue_name, project)
return _basic_message(message[0], now)
def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_SUBSCRIPTIONS_PER_PAGE):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': utils.to_oid(marker)}
projection = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1, 'c': 1}
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
now = timeutils.utcnow_ts()
def normalizer(record):
marker_name['next'] = record['_id']
return _basic_subscription(record, now)
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_SUBSCRIPTIONS_PER_PAGE):
query = {'s': queue, 'p': project}
if marker is not None:
query['_id'] = {'$gt': utils.to_oid(marker)}
projection = {'s': 1, 'u': 1, 't': 1, 'p': 1, 'o': 1, '_id': 1, 'c': 1}
cursor = self._collection.find(query, projection=projection)
cursor = cursor.limit(limit).sort('_id')
marker_name = {}
now = timeutils.utcnow_ts()
def normalizer(record):
marker_name['next'] = record['_id']
return _basic_subscription(record, now)
yield utils.HookedCursor(cursor, normalizer)
yield marker_name and marker_name['next']
def get(self, queue_name, message_id, project=None):
mid = utils.to_oid(message_id)
if mid is None:
raise errors.MessageDoesNotExist(message_id, queue_name,
project)
now = timeutils.utcnow_ts()
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
collection = self._collection(queue_name, project)
message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))
if not message:
raise errors.MessageDoesNotExist(message_id, queue_name,
project)
return _basic_message(message[0], now)
def delete(self, queue, subscription_id, project=None):
self._collection.delete_one({'_id': utils.to_oid(subscription_id),
'p': project,
's': queue})
def delete(self, queue, subscription_id, project=None):
self._collection.remove({'_id': utils.to_oid(subscription_id),
'p': project}, w=0)
def exists(self, queue, subscription_id, project=None):
return self._collection.find_one({'_id': utils.to_oid(subscription_id),
'p': project}) is not None
