def create(self, queue, subscriber, ttl, options, project=None): subscription_id = uuidutils.generate_uuid() subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() expires = now + ttl confirmed = 0 subscription = {'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': self._packer(options), 'p': project, 'c': confirmed} try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: if not self._is_duplicated_subscriber(subscriber, queue, project): pipe.zadd(subset_key, {subscription_id: 1}).hmset( subscription_id, subscription) pipe.expire(subscription_id, ttl) pipe.execute() else: return None return subscription_id except redis.exceptions.ResponseError: return None
def _is_duplicated_subscriber(self, subscriber, queue, project): """Check if the subscriber is existing or not. Given the limitation of Redis' expires(), it's hard to auto expire subscriber from the set and subscription id from the sorted set, so this method is used to do a ugly duplication check when adding a new subscription so that we don't need the set for subscriber. And as a side effect, this method will remove the unreachable subscription's id from the sorted set. """ subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) try: sub_ids = (q for q in self._client.zrange(subset_key, 0, -1)) for s_id in sub_ids: subscription = self._client.hmget(s_id, ['s', 'u', 't', 'o', 'c']) if subscription == [None, None, None, None, None]: # NOTE(flwang): Under this check, that means the # subscription has been expired. So redis can't get # the subscription but the id is still there. So let's # delete the id for clean up. self._client.zrem(subset_key, s_id) if subscription[1].decode() == subscriber: return True return False except redis.exceptions.ResponseError: return True
def create(self, queue, subscriber, ttl, options, project=None): subscription_id = str(uuid.uuid4()) subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() ttl = int(ttl) expires = now + ttl subscription = {'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': options, 'p': project} if not self._queue_ctrl.exists(queue, project): raise errors.QueueDoesNotExist(queue, project) try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: pipe.zadd(subset_key, 1, subscription_id).hmset(subscription_id, subscription) pipe.execute() return subscription_id except redis.exceptions.ResponseError: return None
def list(self, queue, project=None, marker=None, limit=10): client = self._client subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) marker = utils.scope_queue_name(marker, project) rank = client.zrank(subset_key, marker) start = rank + 1 if rank else 0 cursor = (q for q in client.zrange(subset_key, start, start + limit - 1)) marker_next = {} def denormalizer(record, sid): ret = { 'id': sid, 'source': record[0], 'subscriber': record[1], 'ttl': record[2], 'options': json.loads(record[3]), } marker_next['next'] = sid return ret yield utils.SubscriptionListCursor(self._client, cursor, denormalizer) yield marker_next and marker_next['next']
def _is_duplicated_subscriber(self, subscriber, queue, project): """Check if the subscriber is existing or not. Given the limitation of Redis' expires(), it's hard to auto expire subscriber from the set and subscription id from the sorted set, so this method is used to do a ugly duplication check when adding a new subscription so that we don't need the set for subscriber. And as a side effect, this method will remove the unreachable subscription's id from the sorted set. """ subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) try: sub_ids = (q for q in self._client.zrange(subset_key, 0, -1)) for s_id in sub_ids: subscription = self._client.hmget(s_id, ['s', 'u', 't', 'o', 'c']) if subscription == [None, None, None, None, None]: # NOTE(flwang): Under this check, that means the # subscription has been expired. So redis can't get # the subscription but the id is still there. So let's # delete the id for clean up. self._client.zrem(subset_key, s_id) if subscription[1] == subscriber: return True return False except redis.exceptions.ResponseError: return True
def list(self, queue, project=None, marker=None, limit=10): client = self._client subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) rank = client.zrank(subset_key, marker) start = rank + 1 if rank is not None else 0 cursor = (q for q in client.zrange(subset_key, start, start + limit - 1)) marker_next = {} def denormalizer(record, sid): now = timeutils.utcnow_ts() ttl = int(record[2]) expires = int(record[3]) created = expires - ttl ret = { 'id': sid, 'source': record[0], 'subscriber': record[1], 'ttl': ttl, 'age': now - created, 'options': self._unpacker(record[4]), } marker_next['next'] = sid return ret yield utils.SubscriptionListCursor(self._client, cursor, denormalizer) yield marker_next and marker_next['next']
def create(self, queue, subscriber, ttl, options, project=None): subscription_id = uuidutils.generate_uuid() subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() expires = now + ttl confirmed = 0 subscription = {'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': self._packer(options), 'p': project, 'c': confirmed} try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: if not self._is_duplicated_subscriber(subscriber, queue, project): pipe.zadd(subset_key, {subscription_id: 1}).hmset( subscription_id, subscription) pipe.expire(subscription_id, ttl) pipe.execute() else: return None return subscription_id except redis.exceptions.ResponseError: return None
def list(self, queue, project=None, marker=None, limit=10): client = self._client subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) rank = client.zrank(subset_key, marker) start = rank + 1 if rank is not None else 0 cursor = (q for q in client.zrange(subset_key, start, start + limit - 1)) marker_next = {} def denormalizer(record, sid): now = timeutils.utcnow_ts() ttl = int(record[2]) expires = int(record[3]) created = expires - ttl ret = { 'id': sid, 'source': record[0], 'subscriber': record[1], 'ttl': ttl, 'age': now - created, 'options': self._unpacker(record[4]), } marker_next['next'] = sid return ret yield utils.SubscriptionListCursor(self._client, cursor, denormalizer) yield marker_next and marker_next['next']
def list(self, queue, project=None, marker=None, limit=10): client = self._client subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) marker = utils.scope_queue_name(marker, project) rank = client.zrank(subset_key, marker) start = rank + 1 if rank else 0 cursor = (q for q in client.zrange(subset_key, start, start + limit - 1)) marker_next = {} def denormalizer(record, sid): ret = { 'id': sid, 'source': record[0], 'subscriber': record[1], 'ttl': record[2], 'options': json.loads(record[3]), } marker_next['next'] = sid return ret yield utils.SubscriptionListCursor(self._client, cursor, denormalizer) yield marker_next and marker_next['next']
def create(self, queue, subscriber, ttl, options, project=None): subscription_id = str(uuid.uuid4()) subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() ttl = int(ttl) expires = now + ttl subscription = { 'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': options, 'p': project } if not self._queue_ctrl.exists(queue, project): raise errors.QueueDoesNotExist(queue, project) try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: pipe.zadd(subset_key, 1, subscription_id).hmset(subscription_id, subscription) pipe.execute() return subscription_id except redis.exceptions.ResponseError: return None
def delete(self, queue, subscription_id, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) # NOTE(prashanthr_): Pipelining is used to mitigate race conditions with self._client.pipeline() as pipe: pipe.zrem(subset_key, subscription_id) pipe.delete(subscription_id) pipe.execute()
def delete(self, queue, subscription_id, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) # NOTE(prashanthr_): Pipelining is used to mitigate race conditions with self._client.pipeline() as pipe: pipe.zrem(subset_key, subscription_id) pipe.delete(subscription_id) pipe.execute()
def get_with_subscriber(self, queue, subscriber, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) sub_ids = (q for q in self._client.zrange(subset_key, 0, -1)) for s_id in sub_ids: subscription = self._client.hmget(s_id, ['s', 'u', 't', 'o', 'c']) if subscription[1] == subscriber: subscription = SubscriptionEnvelope.from_redis( s_id, self._client) now = timeutils.utcnow_ts() return subscription.to_basic(now)
def get_with_subscriber(self, queue, subscriber, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) sub_ids = (q for q in self._client.zrange(subset_key, 0, -1)) for s_id in sub_ids: subscription = self._client.hmget(s_id, ['s', 'u', 't', 'o', 'c']) if subscription[1] == subscriber: subscription = SubscriptionEnvelope.from_redis(s_id, self._client) now = timeutils.utcnow_ts() return subscription.to_basic(now)
def exists(self, queue, subscription_id, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) return self._client.zrank(subset_key, subscription_id) is not None
def exists(self, queue, subscription_id, project=None): subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) return self._client.zrank(subset_key, subscription_id) is not None