def check_repeaters():
start = datetime.utcnow()
cutoff = start + CHECK_REPEATERS_INTERVAL
redis_client = get_redis_client().client.get_client()
# Timeout for slightly less than periodic check
check_repeater_lock = redis_client.lock(
CHECK_REPEATERS_KEY, timeout=CHECK_REPEATERS_INTERVAL.seconds - 10)
if not check_repeater_lock.acquire(blocking=False):
return
for record in iterate_repeat_records(start):
now = datetime.utcnow()
lock_key = _get_repeat_record_lock_key(record)
if now > cutoff:
break
lock = redis_client.lock(lock_key, timeout=60 * 60 * 48)
if not lock.acquire(blocking=False):
continue
process_repeat_record.delay(record)
try:
check_repeater_lock.release()
except LockError:
# Ignore if already released
pass
def check_repeaters():
start = datetime.utcnow()
six_hours_sec = 6 * 60 * 60
six_hours_later = start + timedelta(seconds=six_hours_sec)
# Long timeout to allow all waiting repeat records to be iterated
check_repeater_lock = get_redis_lock(
CHECK_REPEATERS_KEY,
timeout=six_hours_sec,
name=CHECK_REPEATERS_KEY,
)
if not check_repeater_lock.acquire(blocking=False):
datadog_counter("commcare.repeaters.check.locked_out")
return
try:
with datadog_bucket_timer(
"commcare.repeaters.check.processing",
tags=[],
timing_buckets=_check_repeaters_buckets,
):
for record in iterate_repeat_records(start):
if datetime.utcnow() > six_hours_later:
_soft_assert(False, "I've been iterating repeat records for six hours. I quit!")
break
datadog_counter("commcare.repeaters.check.attempt_forward")
record.attempt_forward_now()
finally:
check_repeater_lock.release()
def check_repeaters():
start = datetime.utcnow()
six_hours_sec = 6 * 60 * 60
six_hours_later = start + timedelta(seconds=six_hours_sec)
# Long timeout to allow all waiting repeat records to be iterated
check_repeater_lock = get_redis_lock(
CHECK_REPEATERS_KEY,
timeout=six_hours_sec,
name=CHECK_REPEATERS_KEY,
)
if not check_repeater_lock.acquire(blocking=False):
metrics_counter("commcare.repeaters.check.locked_out")
return
try:
with datadog_bucket_timer(
"commcare.repeaters.check.processing",
tags=[],
timing_buckets=_check_repeaters_buckets,
):
for record in iterate_repeat_records(start):
if datetime.utcnow() > six_hours_later:
_soft_assert(
False,
"I've been iterating repeat records for six hours. I quit!"
)
break
metrics_counter("commcare.repeaters.check.attempt_forward")
record.attempt_forward_now()
finally:
check_repeater_lock.release()
def check_repeaters():
start = datetime.utcnow()
cutoff = start + CHECK_REPEATERS_INTERVAL
redis_client = get_redis_client().client.get_client()
# Timeout for slightly less than periodic check
check_repeater_lock = redis_client.lock(
CHECK_REPEATERS_KEY,
timeout=CHECK_REPEATERS_INTERVAL.seconds - 10
)
if not check_repeater_lock.acquire(blocking=False):
return
for record in iterate_repeat_records(start):
now = datetime.utcnow()
lock_key = _get_repeat_record_lock_key(record)
if now > cutoff:
break
lock = redis_client.lock(lock_key, timeout=60 * 60 * 48)
if not lock.acquire(blocking=False):
continue
process_repeat_record.delay(record)
try:
check_repeater_lock.release()
except LockError:
# Ignore if already released
pass
def check_repeaters():
start = datetime.utcnow()
twentythree_hours_sec = 23 * 60 * 60
twentythree_hours_later = start + timedelta(hours=23)
# Long timeout to allow all waiting repeat records to be iterated
check_repeater_lock = get_redis_lock(
CHECK_REPEATERS_KEY,
timeout=twentythree_hours_sec,
name=CHECK_REPEATERS_KEY,
)
if not check_repeater_lock.acquire(blocking=False):
metrics_counter("commcare.repeaters.check.locked_out")
return
try:
with metrics_histogram_timer(
"commcare.repeaters.check.processing",
timing_buckets=_check_repeaters_buckets,
):
for record in iterate_repeat_records(start):
if not _soft_assert(
datetime.utcnow() < twentythree_hours_later,
"I've been iterating repeat records for 23 hours. I quit!"
):
break
metrics_counter("commcare.repeaters.check.attempt_forward")
record.attempt_forward_now()
else:
iterating_time = datetime.utcnow() - start
_soft_assert(
iterating_time < timedelta(hours=6),
f"It took {iterating_time} to iterate repeat records.")
finally:
check_repeater_lock.release()
def check_repeaters():
start = datetime.utcnow()
six_hours_sec = 6 * 60 * 60
six_hours_later = start + timedelta(seconds=six_hours_sec)
# Long timeout to allow all waiting repeat records to be iterated
check_repeater_lock = get_redis_lock(
CHECK_REPEATERS_KEY,
timeout=six_hours_sec,
name=CHECK_REPEATERS_KEY,
)
if not check_repeater_lock.acquire(blocking=False):
return
try:
for record in iterate_repeat_records(start):
if datetime.utcnow() > six_hours_later:
_soft_assert(False, "I've been iterating repeat records for six hours. I quit!")
break
if acquire_redis_lock(record):
record.attempt_forward_now()
finally:
check_repeater_lock.release()
def test_iterate_repeat_records(self):
records = list(iterate_repeat_records(datetime.utcnow(), chunk_size=2))
self.assertEqual(len(records), 4) # Should grab all but the succeeded one