def deduper(query_id: Optional[str]) -> Iterator[bool]: """ A simple redis distributed lock on a query_id to prevent multiple concurrent queries running with the same id. Blocks subsequent queries until the first is finished. When used in conjunction with caching this means that the subsequent queries can then use the cached result from the first query. """ unlock = ''' if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end ''' if query_id is None: yield False else: lock = '{}{}'.format(query_lock_prefix, query_id) nonce = uuid.uuid4() try: is_dupe = False while not rds.set(lock, nonce, nx=True, ex=max_query_duration_s): is_dupe = True time.sleep(0.01) yield is_dupe finally: rds.eval(unlock, 1, lock, nonce)
def set_project_needs_final( project_id: int, state_name: Optional[ReplacerState] ) -> Optional[bool]: return redis_client.set( get_project_needs_final_key(project_id, state_name), True, ex=settings.REPLACER_KEY_TTL, )
def test_offset_already_processed(self) -> None: """ Don't process an offset that already exists in Redis. """ set_config("skip_seen_offsets", True) self.event["project_id"] = self.project_id self.event["group_id"] = 1 self.event["primary_hash"] = "a" * 32 write_unprocessed_events(self.storage, [self.event]) key = f"replacement:{CONSUMER_GROUP}:errors:1" redis_client.set(key, 42) old_offset: Message[KafkaPayload] = Message( Partition(Topic("replacements"), 1), 41, KafkaPayload( None, json.dumps(( 2, ReplacementType.END_UNMERGE, {}, )).encode("utf-8"), [], ), datetime.now(), ) same_offset: Message[KafkaPayload] = Message( Partition(Topic("replacements"), 1), 42, KafkaPayload( None, json.dumps(( 2, ReplacementType.END_UNMERGE, {}, )).encode("utf-8"), [], ), datetime.now(), ) assert self.replacer.process_message(old_offset) is None assert self.replacer.process_message(same_offset) is None
def _check_timing_and_write_to_redis(self, replacement: Replacement, start_time: float) -> None: """ Write the offset just processed to Redis if execution took longer than the threshold. Also store the offset locally to avoid Read calls to Redis. If the Consumer dies while the insert query for the message was being executed, the most recently executed offset would be present in Redis. """ if (time.time() - start_time) < settings.REPLACER_PROCESSING_TIMEOUT_THRESHOLD: return message_metadata = replacement.get_message_metadata() key = self._build_topic_group_index_key(message_metadata) redis_client.set( key, message_metadata.offset, ex=settings.REPLACER_PROCESSING_TIMEOUT_THRESHOLD_KEY_TTL, ) self.__last_offset_processed_per_partition[ key] = message_metadata.offset
def set_project_needs_final(project_id): return redis_client.set(get_project_needs_final_key(project_id), True, ex=settings.REPLACER_KEY_TTL)
def set_result(query_id: str, result: Mapping[str, Optional[Any]]) -> Any: timeout = get_config('cache_expiry_sec', 1) key = '{}{}'.format(query_cache_prefix, query_id) return rds.set(key, json.dumps(result), ex=timeout)
def set_result(query_id, result): timeout = get_config('cache_expiry_sec', 1) key = '{}{}'.format(query_cache_prefix, query_id) return rds.set(key, json.dumps(result), ex=timeout)