def deduper(query_id: Optional[str]) -> Iterator[bool]:
"""
A simple redis distributed lock on a query_id to prevent multiple
concurrent queries running with the same id. Blocks subsequent
queries until the first is finished.
When used in conjunction with caching this means that the subsequent
queries can then use the cached result from the first query.
"""
unlock = '''
if redis.call('get', KEYS[1]) == ARGV[1]
then
return redis.call('del', KEYS[1])
else
return 0
end
'''
if query_id is None:
yield False
else:
lock = '{}{}'.format(query_lock_prefix, query_id)
nonce = uuid.uuid4()
try:
is_dupe = False
while not rds.set(lock, nonce, nx=True, ex=max_query_duration_s):
is_dupe = True
time.sleep(0.01)
yield is_dupe
finally:
rds.eval(unlock, 1, lock, nonce)
def set_project_needs_final(
project_id: int, state_name: Optional[ReplacerState]
) -> Optional[bool]:
return redis_client.set(
get_project_needs_final_key(project_id, state_name),
True,
ex=settings.REPLACER_KEY_TTL,
)
def test_offset_already_processed(self) -> None:
"""
Don't process an offset that already exists in Redis.
"""
set_config("skip_seen_offsets", True)
self.event["project_id"] = self.project_id
self.event["group_id"] = 1
self.event["primary_hash"] = "a" * 32
write_unprocessed_events(self.storage, [self.event])
key = f"replacement:{CONSUMER_GROUP}:errors:1"
redis_client.set(key, 42)
old_offset: Message[KafkaPayload] = Message(
Partition(Topic("replacements"), 1),
41,
KafkaPayload(
None,
json.dumps((
2,
ReplacementType.END_UNMERGE,
{},
)).encode("utf-8"),
[],
),
datetime.now(),
)
same_offset: Message[KafkaPayload] = Message(
Partition(Topic("replacements"), 1),
42,
KafkaPayload(
None,
json.dumps((
2,
ReplacementType.END_UNMERGE,
{},
)).encode("utf-8"),
[],
),
datetime.now(),
)
assert self.replacer.process_message(old_offset) is None
assert self.replacer.process_message(same_offset) is None
def _check_timing_and_write_to_redis(self, replacement: Replacement,
start_time: float) -> None:
"""
Write the offset just processed to Redis if execution took longer than the threshold.
Also store the offset locally to avoid Read calls to Redis.
If the Consumer dies while the insert query for the message was being executed,
the most recently executed offset would be present in Redis.
"""
if (time.time() -
start_time) < settings.REPLACER_PROCESSING_TIMEOUT_THRESHOLD:
return
message_metadata = replacement.get_message_metadata()
key = self._build_topic_group_index_key(message_metadata)
redis_client.set(
key,
message_metadata.offset,
ex=settings.REPLACER_PROCESSING_TIMEOUT_THRESHOLD_KEY_TTL,
)
self.__last_offset_processed_per_partition[
key] = message_metadata.offset
def set_project_needs_final(project_id):
return redis_client.set(get_project_needs_final_key(project_id),
True,
ex=settings.REPLACER_KEY_TTL)
def set_result(query_id: str, result: Mapping[str, Optional[Any]]) -> Any:
timeout = get_config('cache_expiry_sec', 1)
key = '{}{}'.format(query_cache_prefix, query_id)
return rds.set(key, json.dumps(result), ex=timeout)
def set_result(query_id, result):
timeout = get_config('cache_expiry_sec', 1)
key = '{}{}'.format(query_cache_prefix, query_id)
return rds.set(key, json.dumps(result), ex=timeout)
