def test_descope_messages_set(self): key = utils.scope_message_ids_set('my-q') self.assertEqual(('my-q', None), utils.descope_message_ids_set(key)) key = utils.scope_message_ids_set('my-q', '123') self.assertEqual(('my-q', '123'), utils.descope_message_ids_set(key)) key = utils.scope_message_ids_set(None, '123') self.assertEqual((None, '123'), utils.descope_message_ids_set(key)) key = utils.scope_message_ids_set() self.assertEqual((None, None), utils.descope_message_ids_set(key))
def gc(self): """Garbage-collect expired message data. Not all message data can be automatically expired. This method cleans up the remainder. :returns: Number of messages removed """ claim_ctrl = self.driver.claim_controller client = self._client num_removed = 0 offset_msgsets = 0 while True: # NOTE(kgriffs): Iterate across all message sets; there will # be one set of message IDs per queue. msgset_keys = client.zrange(MSGSET_INDEX_KEY, offset_msgsets, offset_msgsets + GC_BATCH_SIZE - 1) if not msgset_keys: break offset_msgsets += len(msgset_keys) for msgset_key in msgset_keys: msgset_key = encodeutils.safe_decode(msgset_key) # NOTE(kgriffs): Drive the claim controller GC from # here, because we already know the queue and project # scope. queue, project = utils.descope_message_ids_set(msgset_key) claim_ctrl._gc(queue, project) offset_mids = 0 while True: # NOTE(kgriffs): Look up each message in the message set, # see if it has expired, and if so, remove it from msgset. mids = client.zrange(msgset_key, offset_mids, offset_mids + GC_BATCH_SIZE - 1) if not mids: break offset_mids += len(mids) # NOTE(kgriffs): If redis expired the message, it will # not exist, so all we have to do is remove mid from # the msgset collection. with client.pipeline() as pipe: for mid in mids: pipe.exists(mid) mid_exists_flags = pipe.execute() with client.pipeline() as pipe: for mid, exists in zip(mids, mid_exists_flags): if not exists: pipe.zrem(msgset_key, mid) num_removed += 1 pipe.execute() return num_removed
def gc(self): """Garbage-collect expired message data. Not all message data can be automatically expired. This method cleans up the remainder. :returns: Number of messages removed """ client = self._client num_removed = 0 offset_msgsets = 0 while True: # NOTE(kgriffs): Iterate across all message sets; there will # be one set of message IDs per queue. msgset_keys = client.zrange(MSGSET_INDEX_KEY, offset_msgsets, offset_msgsets + GC_BATCH_SIZE - 1) if not msgset_keys: break offset_msgsets += len(msgset_keys) for msgset_key in msgset_keys: msgset_key = encodeutils.safe_decode(msgset_key) # NOTE(kgriffs): Drive the claim controller GC from # here, because we already know the queue and project # scope. queue, project = utils.descope_message_ids_set(msgset_key) self._claim_ctrl._gc(queue, project) offset_mids = 0 while True: # NOTE(kgriffs): Look up each message in the message set, # see if it has expired, and if so, remove it from msgset. mids = client.zrange(msgset_key, offset_mids, offset_mids + GC_BATCH_SIZE - 1) if not mids: break offset_mids += len(mids) # NOTE(kgriffs): If redis expired the message, it will # not exist, so all we have to do is remove mid from # the msgset collection. with client.pipeline() as pipe: for mid in mids: pipe.exists(mid) mid_exists_flags = pipe.execute() with client.pipeline() as pipe: for mid, exists in zip(mids, mid_exists_flags): if not exists: pipe.zrem(msgset_key, mid) num_removed += 1 pipe.execute() return num_removed