def check_repeaters(): start = datetime.utcnow() cutoff = start + CHECK_REPEATERS_INTERVAL redis_client = get_redis_client().client.get_client() # Timeout for slightly less than periodic check check_repeater_lock = redis_client.lock( CHECK_REPEATERS_KEY, timeout=CHECK_REPEATERS_INTERVAL.seconds - 10) if not check_repeater_lock.acquire(blocking=False): return for record in iterate_repeat_records(start): now = datetime.utcnow() lock_key = _get_repeat_record_lock_key(record) if now > cutoff: break lock = redis_client.lock(lock_key, timeout=60 * 60 * 48) if not lock.acquire(blocking=False): continue process_repeat_record.delay(record) try: check_repeater_lock.release() except LockError: # Ignore if already released pass
def check_repeaters(): start = datetime.utcnow() six_hours_sec = 6 * 60 * 60 six_hours_later = start + timedelta(seconds=six_hours_sec) # Long timeout to allow all waiting repeat records to be iterated check_repeater_lock = get_redis_lock( CHECK_REPEATERS_KEY, timeout=six_hours_sec, name=CHECK_REPEATERS_KEY, ) if not check_repeater_lock.acquire(blocking=False): datadog_counter("commcare.repeaters.check.locked_out") return try: with datadog_bucket_timer( "commcare.repeaters.check.processing", tags=[], timing_buckets=_check_repeaters_buckets, ): for record in iterate_repeat_records(start): if datetime.utcnow() > six_hours_later: _soft_assert(False, "I've been iterating repeat records for six hours. I quit!") break datadog_counter("commcare.repeaters.check.attempt_forward") record.attempt_forward_now() finally: check_repeater_lock.release()
def check_repeaters(): start = datetime.utcnow() six_hours_sec = 6 * 60 * 60 six_hours_later = start + timedelta(seconds=six_hours_sec) # Long timeout to allow all waiting repeat records to be iterated check_repeater_lock = get_redis_lock( CHECK_REPEATERS_KEY, timeout=six_hours_sec, name=CHECK_REPEATERS_KEY, ) if not check_repeater_lock.acquire(blocking=False): metrics_counter("commcare.repeaters.check.locked_out") return try: with datadog_bucket_timer( "commcare.repeaters.check.processing", tags=[], timing_buckets=_check_repeaters_buckets, ): for record in iterate_repeat_records(start): if datetime.utcnow() > six_hours_later: _soft_assert( False, "I've been iterating repeat records for six hours. I quit!" ) break metrics_counter("commcare.repeaters.check.attempt_forward") record.attempt_forward_now() finally: check_repeater_lock.release()
def check_repeaters(): start = datetime.utcnow() cutoff = start + CHECK_REPEATERS_INTERVAL redis_client = get_redis_client().client.get_client() # Timeout for slightly less than periodic check check_repeater_lock = redis_client.lock( CHECK_REPEATERS_KEY, timeout=CHECK_REPEATERS_INTERVAL.seconds - 10 ) if not check_repeater_lock.acquire(blocking=False): return for record in iterate_repeat_records(start): now = datetime.utcnow() lock_key = _get_repeat_record_lock_key(record) if now > cutoff: break lock = redis_client.lock(lock_key, timeout=60 * 60 * 48) if not lock.acquire(blocking=False): continue process_repeat_record.delay(record) try: check_repeater_lock.release() except LockError: # Ignore if already released pass
def check_repeaters(): start = datetime.utcnow() twentythree_hours_sec = 23 * 60 * 60 twentythree_hours_later = start + timedelta(hours=23) # Long timeout to allow all waiting repeat records to be iterated check_repeater_lock = get_redis_lock( CHECK_REPEATERS_KEY, timeout=twentythree_hours_sec, name=CHECK_REPEATERS_KEY, ) if not check_repeater_lock.acquire(blocking=False): metrics_counter("commcare.repeaters.check.locked_out") return try: with metrics_histogram_timer( "commcare.repeaters.check.processing", timing_buckets=_check_repeaters_buckets, ): for record in iterate_repeat_records(start): if not _soft_assert( datetime.utcnow() < twentythree_hours_later, "I've been iterating repeat records for 23 hours. I quit!" ): break metrics_counter("commcare.repeaters.check.attempt_forward") record.attempt_forward_now() else: iterating_time = datetime.utcnow() - start _soft_assert( iterating_time < timedelta(hours=6), f"It took {iterating_time} to iterate repeat records.") finally: check_repeater_lock.release()
def check_repeaters(): start = datetime.utcnow() six_hours_sec = 6 * 60 * 60 six_hours_later = start + timedelta(seconds=six_hours_sec) # Long timeout to allow all waiting repeat records to be iterated check_repeater_lock = get_redis_lock( CHECK_REPEATERS_KEY, timeout=six_hours_sec, name=CHECK_REPEATERS_KEY, ) if not check_repeater_lock.acquire(blocking=False): return try: for record in iterate_repeat_records(start): if datetime.utcnow() > six_hours_later: _soft_assert(False, "I've been iterating repeat records for six hours. I quit!") break if acquire_redis_lock(record): record.attempt_forward_now() finally: check_repeater_lock.release()
def test_iterate_repeat_records(self): records = list(iterate_repeat_records(datetime.utcnow(), chunk_size=2)) self.assertEqual(len(records), 4) # Should grab all but the succeeded one