def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
r = get_redis_connection()
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True,
timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
for run in runs:
run_key = '%d:%d' % (run.id, datetime_to_epoch(run.timeout_on))
# check whether we have already queued this timeout
pipe = r.pipeline()
pipe.sismember(timezone.now().strftime(FLOW_TIMEOUT_KEY), run_key)
pipe.sismember(
(timezone.now() - timedelta(days=1)).strftime(FLOW_TIMEOUT_KEY),
run_key)
(queued_today, queued_yesterday) = pipe.execute()
# if not, add a task to handle the timeout
if not queued_today and not queued_yesterday:
push_task(
run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=TIMEOUT_EVENT, run=run.id,
timeout_on=run.timeout_on))
# tag this run as being worked on so we don't double queue
pipe = r.pipeline()
sent_key = timezone.now().strftime(FLOW_TIMEOUT_KEY)
pipe.sadd(sent_key, run_key)
pipe.expire(sent_key, 86400)
pipe.execute()
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True,
timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
queued_timeouts = QueueRecord(
'flow_timeouts', lambda r: '%d:%d' %
(r.id, datetime_to_epoch(r.timeout_on)))
for run in runs:
# ignore any run which was locked by previous calls to this task
if not queued_timeouts.is_queued(run):
try:
task_payload = dict(type=TIMEOUT_EVENT,
run=run.id,
timeout_on=run.timeout_on)
push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
task_payload)
queued_timeouts.set_queued([run])
except Exception: # pragma: no cover
logger.error("Error queuing timeout task for run #%d" % run.id,
exc_info=True)