def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = uuidutils.generate_uuid()
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
expires = now + ttl
confirmed = 0
subscription = {'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': self._packer(options),
'p': project,
'c': confirmed}
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
if not self._is_duplicated_subscriber(subscriber,
queue,
project):
pipe.zadd(subset_key, {subscription_id: 1}).hmset(
subscription_id, subscription)
pipe.expire(subscription_id, ttl)
pipe.execute()
else:
return None
return subscription_id
except redis.exceptions.ResponseError:
return None
def _is_duplicated_subscriber(self, subscriber, queue, project):
"""Check if the subscriber is existing or not.
Given the limitation of Redis' expires(), it's hard to auto expire
subscriber from the set and subscription id from the sorted set, so
this method is used to do a ugly duplication check when adding a new
subscription so that we don't need the set for subscriber. And as a
side effect, this method will remove the unreachable subscription's id
from the sorted set.
"""
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
try:
sub_ids = (q for q in self._client.zrange(subset_key, 0, -1))
for s_id in sub_ids:
subscription = self._client.hmget(s_id,
['s', 'u', 't', 'o', 'c'])
if subscription == [None, None, None, None, None]:
# NOTE(flwang): Under this check, that means the
# subscription has been expired. So redis can't get
# the subscription but the id is still there. So let's
# delete the id for clean up.
self._client.zrem(subset_key, s_id)
if subscription[1].decode() == subscriber:
return True
return False
except redis.exceptions.ResponseError:
return True
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = str(uuid.uuid4())
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
subscription = {'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project}
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.zadd(subset_key, 1,
subscription_id).hmset(subscription_id,
subscription)
pipe.execute()
return subscription_id
except redis.exceptions.ResponseError:
return None
def list(self, queue, project=None, marker=None, limit=10):
client = self._client
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
marker = utils.scope_queue_name(marker, project)
rank = client.zrank(subset_key, marker)
start = rank + 1 if rank else 0
cursor = (q for q in client.zrange(subset_key, start,
start + limit - 1))
marker_next = {}
def denormalizer(record, sid):
ret = {
'id': sid,
'source': record[0],
'subscriber': record[1],
'ttl': record[2],
'options': json.loads(record[3]),
}
marker_next['next'] = sid
return ret
yield utils.SubscriptionListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def _is_duplicated_subscriber(self, subscriber, queue, project):
"""Check if the subscriber is existing or not.
Given the limitation of Redis' expires(), it's hard to auto expire
subscriber from the set and subscription id from the sorted set, so
this method is used to do a ugly duplication check when adding a new
subscription so that we don't need the set for subscriber. And as a
side effect, this method will remove the unreachable subscription's id
from the sorted set.
"""
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
try:
sub_ids = (q for q in self._client.zrange(subset_key, 0, -1))
for s_id in sub_ids:
subscription = self._client.hmget(s_id,
['s', 'u', 't', 'o', 'c'])
if subscription == [None, None, None, None, None]:
# NOTE(flwang): Under this check, that means the
# subscription has been expired. So redis can't get
# the subscription but the id is still there. So let's
# delete the id for clean up.
self._client.zrem(subset_key, s_id)
if subscription[1] == subscriber:
return True
return False
except redis.exceptions.ResponseError:
return True
def list(self, queue, project=None, marker=None, limit=10):
client = self._client
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
rank = client.zrank(subset_key, marker)
start = rank + 1 if rank is not None else 0
cursor = (q
for q in client.zrange(subset_key, start, start + limit - 1))
marker_next = {}
def denormalizer(record, sid):
now = timeutils.utcnow_ts()
ttl = int(record[2])
expires = int(record[3])
created = expires - ttl
ret = {
'id': sid,
'source': record[0],
'subscriber': record[1],
'ttl': ttl,
'age': now - created,
'options': self._unpacker(record[4]),
}
marker_next['next'] = sid
return ret
yield utils.SubscriptionListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = uuidutils.generate_uuid()
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
expires = now + ttl
confirmed = 0
subscription = {'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': self._packer(options),
'p': project,
'c': confirmed}
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
if not self._is_duplicated_subscriber(subscriber,
queue,
project):
pipe.zadd(subset_key, {subscription_id: 1}).hmset(
subscription_id, subscription)
pipe.expire(subscription_id, ttl)
pipe.execute()
else:
return None
return subscription_id
except redis.exceptions.ResponseError:
return None
def list(self, queue, project=None, marker=None, limit=10):
client = self._client
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
rank = client.zrank(subset_key, marker)
start = rank + 1 if rank is not None else 0
cursor = (q for q in client.zrange(subset_key, start,
start + limit - 1))
marker_next = {}
def denormalizer(record, sid):
now = timeutils.utcnow_ts()
ttl = int(record[2])
expires = int(record[3])
created = expires - ttl
ret = {
'id': sid,
'source': record[0],
'subscriber': record[1],
'ttl': ttl,
'age': now - created,
'options': self._unpacker(record[4]),
}
marker_next['next'] = sid
return ret
yield utils.SubscriptionListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def list(self, queue, project=None, marker=None, limit=10):
client = self._client
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
marker = utils.scope_queue_name(marker, project)
rank = client.zrank(subset_key, marker)
start = rank + 1 if rank else 0
cursor = (q
for q in client.zrange(subset_key, start, start + limit - 1))
marker_next = {}
def denormalizer(record, sid):
ret = {
'id': sid,
'source': record[0],
'subscriber': record[1],
'ttl': record[2],
'options': json.loads(record[3]),
}
marker_next['next'] = sid
return ret
yield utils.SubscriptionListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = str(uuid.uuid4())
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
subscription = {
'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project
}
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.zadd(subset_key, 1,
subscription_id).hmset(subscription_id, subscription)
pipe.execute()
return subscription_id
except redis.exceptions.ResponseError:
return None
def delete(self, queue, subscription_id, project=None):
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
# NOTE(prashanthr_): Pipelining is used to mitigate race conditions
with self._client.pipeline() as pipe:
pipe.zrem(subset_key, subscription_id)
pipe.delete(subscription_id)
pipe.execute()
def delete(self, queue, subscription_id, project=None):
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
# NOTE(prashanthr_): Pipelining is used to mitigate race conditions
with self._client.pipeline() as pipe:
pipe.zrem(subset_key, subscription_id)
pipe.delete(subscription_id)
pipe.execute()
def get_with_subscriber(self, queue, subscriber, project=None):
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
sub_ids = (q for q in self._client.zrange(subset_key, 0, -1))
for s_id in sub_ids:
subscription = self._client.hmget(s_id, ['s', 'u', 't', 'o', 'c'])
if subscription[1] == subscriber:
subscription = SubscriptionEnvelope.from_redis(
s_id, self._client)
now = timeutils.utcnow_ts()
return subscription.to_basic(now)
def get_with_subscriber(self, queue, subscriber, project=None):
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
sub_ids = (q for q in self._client.zrange(subset_key, 0, -1))
for s_id in sub_ids:
subscription = self._client.hmget(s_id,
['s', 'u', 't', 'o', 'c'])
if subscription[1] == subscriber:
subscription = SubscriptionEnvelope.from_redis(s_id,
self._client)
now = timeutils.utcnow_ts()
return subscription.to_basic(now)
def exists(self, queue, subscription_id, project=None):
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
return self._client.zrank(subset_key, subscription_id) is not None
def exists(self, queue, subscription_id, project=None):
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
return self._client.zrank(subset_key, subscription_id) is not None