def check_campaigns_task(sched_id=None):
"""
See if any event fires need to be triggered
"""
logger = check_campaigns_task.get_logger()
# get a lock
r = get_redis_connection()
key = 'check_campaigns'
# only do this if we aren't already checking campaigns
if not r.get(key):
with r.lock(key, timeout=3600):
# for each that needs to be fired
for fire in EventFire.objects.filter(
fired=None, scheduled__lte=timezone.now()).select_related(
'contact', 'contact__org'):
try:
push_task(fire.contact.org, HANDLER_QUEUE,
HANDLE_EVENT_TASK,
dict(type=FIRE_EVENT, id=fire.id))
except Exception: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk,
exc_info=True)
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
msg_tasks = pop_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks:
return
if not isinstance(msg_tasks, list):
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock('send_contact_%d' % msg_tasks[0]['contact'], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct('MockMsg', msg_task,
datetime_fields=['modified_on', 'sent_on', 'created_on', 'queued_on', 'next_attempt'])
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(msg_tasks[0]['org'], MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True,
timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
queued_timeouts = QueueRecord(
'flow_timeouts', lambda r: '%d:%d' %
(r.id, datetime_to_epoch(r.timeout_on)))
for run in runs:
# ignore any run which was locked by previous calls to this task
if not queued_timeouts.is_queued(run):
try:
task_payload = dict(type=TIMEOUT_EVENT,
run=run.id,
timeout_on=run.timeout_on)
push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
task_payload)
queued_timeouts.set_queued([run])
except Exception: # pragma: no cover
logger.error("Error queuing timeout task for run #%d" % run.id,
exc_info=True)
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
r = get_redis_connection()
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True,
timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
for run in runs:
run_key = '%d:%d' % (run.id, datetime_to_epoch(run.timeout_on))
# check whether we have already queued this timeout
pipe = r.pipeline()
pipe.sismember(timezone.now().strftime(FLOW_TIMEOUT_KEY), run_key)
pipe.sismember(
(timezone.now() - timedelta(days=1)).strftime(FLOW_TIMEOUT_KEY),
run_key)
(queued_today, queued_yesterday) = pipe.execute()
# if not, add a task to handle the timeout
if not queued_today and not queued_yesterday:
push_task(
run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=TIMEOUT_EVENT, run=run.id,
timeout_on=run.timeout_on))
# tag this run as being worked on so we don't double queue
pipe = r.pipeline()
sent_key = timezone.now().strftime(FLOW_TIMEOUT_KEY)
pipe.sadd(sent_key, run_key)
pipe.expire(sent_key, 86400)
pipe.execute()
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True,
timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
for run in runs:
# move this flow forward via the handler queue
push_task(
run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_campaigns_task():
"""
See if any event fires need to be triggered
"""
# for each that needs to be fired
for fire in EventFire.objects.filter(
fired=None, scheduled__lte=timezone.now()).select_related(
'contact', 'contact__org'):
try:
push_task(fire.contact.org, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=FIRE_EVENT, id=fire.id))
except Exception: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk,
exc_info=True)
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
r = get_redis_connection()
# only do this if we aren't already expiring things
key = 'check_flow_timeouts'
if not r.get(key):
with r.lock(key, timeout=900):
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()).only('id', 'org')
for run in runs:
# move this flow forward via the handler queue
push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
r = get_redis_connection()
# only do this if we aren't already expiring things
key = 'check_flow_timeouts'
if not r.get(key):
with r.lock(key, timeout=900):
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now())
runs = runs.only('id', 'org', 'timeout_on')
for run in runs:
# move this flow forward via the handler queue
push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_campaigns_task():
"""
See if any event fires need to be triggered
"""
from temba.flows.models import Flow
unfired = EventFire.objects.filter(fired=None,
scheduled__lte=timezone.now())
unfired = unfired.values('id', 'event__flow_id')
# group fire events by flow so they can be batched
fire_ids_by_flow_id = defaultdict(list)
for fire in unfired:
fire_ids_by_flow_id[fire['event__flow_id']].append(fire['id'])
# fetch the flows used by all these event fires
flows_by_id = {
flow.id: flow
for flow in Flow.objects.filter(id__in=fire_ids_by_flow_id.keys())
}
queued_fires = QueueRecord('queued_event_fires')
# create queued tasks
for flow_id, fire_ids in six.iteritems(fire_ids_by_flow_id):
flow = flows_by_id[flow_id]
# create sub-batches no no single task is too big
for fire_id_batch in chunk_list(fire_ids, 500):
# ignore any fires which were queued by previous calls to this task but haven't yet been marked as fired
queued_fire_ids = queued_fires.filter_unqueued(fire_id_batch)
if queued_fire_ids:
try:
push_task(flow.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=FIRE_EVENT, fires=queued_fire_ids))
queued_fires.set_queued(queued_fire_ids)
except Exception: # pragma: no cover
fire_ids_str = ','.join(
six.text_type(f) for f in queued_fire_ids)
logger.error("Error queuing campaign event fires: %s" %
fire_ids_str,
exc_info=True)
def check_campaigns_task():
"""
See if any event fires need to be triggered
"""
from temba.flows.models import Flow
unfired = EventFire.objects.filter(
fired=None, scheduled__lte=timezone.now(), event__flow__flow_server_enabled=False
).select_related("event")
unfired = unfired.values("id", "event_id", "event__flow_id")
# group fire events by event so they can be batched
fire_ids_by_event_id = defaultdict(list)
event_flow_map = dict()
for fire in unfired:
event_flow_map[fire["event_id"]] = fire["event__flow_id"]
fire_ids_by_event_id[fire["event_id"]].append(fire["id"])
# fetch the flows used by all these event fires
flows_by_id = {flow.id: flow for flow in Flow.objects.filter(id__in=event_flow_map.values())}
queued_fires = QueueRecord("queued_event_fires")
# create queued tasks
for ev_id, fire_ids in fire_ids_by_event_id.items():
flow_id = event_flow_map[ev_id]
flow = flows_by_id[flow_id]
# create sub-batches no no single task is too big
for fire_id_batch in chunk_list(fire_ids, 500):
# ignore any fires which were queued by previous calls to this task but haven't yet been marked as fired
queued_fire_ids = queued_fires.filter_unqueued(fire_id_batch)
if queued_fire_ids:
try:
push_task(
flow.org_id, Queue.HANDLER, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, fires=queued_fire_ids)
)
queued_fires.set_queued(queued_fire_ids)
except Exception: # pragma: no cover
fire_ids_str = ",".join(str(f) for f in queued_fire_ids)
logger.error("Error queuing campaign event fires: %s" % fire_ids_str, exc_info=True)
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
org_id, msg_tasks = start_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks: # pragma: needs cover
return
if not isinstance(msg_tasks, list): # pragma: needs cover
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct(
"MockMsg",
msg_task,
datetime_fields=[
"modified_on", "sent_on", "created_on", "queued_on",
"next_attempt"
],
)
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# mark this worker as done
complete_task(SEND_MSG_TASK, org_id)
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_flow_timeouts_task():
"""
See if any flow runs have timed out
"""
# find any runs that should have timed out
runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now())
runs = runs.only("id", "org", "timeout_on")
queued_timeouts = QueueRecord("flow_timeouts", lambda r: "%d:%d" % (r.id, datetime_to_epoch(r.timeout_on)))
for run in runs:
# ignore any run which was locked by previous calls to this task
if not queued_timeouts.is_queued(run):
try:
task_payload = dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on)
push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, task_payload)
queued_timeouts.set_queued([run])
except Exception: # pragma: no cover
logger.error("Error queuing timeout task for run #%d" % run.id, exc_info=True)
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
org_id, msg_tasks = start_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks: # pragma: needs cover
return
if not isinstance(msg_tasks, list): # pragma: needs cover
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct(
"MockMsg",
msg_task,
datetime_fields=["modified_on", "sent_on", "created_on", "queued_on", "next_attempt"],
)
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# mark this worker as done
complete_task(SEND_MSG_TASK, org_id)
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_campaigns_task(sched_id=None):
"""
See if any event fires need to be triggered
"""
logger = check_campaigns_task.get_logger()
# get a lock
r = get_redis_connection()
key = 'check_campaigns'
# only do this if we aren't already checking campaigns
if not r.get(key):
with r.lock(key, timeout=3600):
# for each that needs to be fired
for fire in EventFire.objects.filter(fired=None, scheduled__lte=timezone.now()).select_related('contact', 'contact.org'):
try:
push_task(fire.contact.org, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, id=fire.id))
except: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk, exc_info=True)
def post(self, request, *args, **kwargs):
from temba.triggers.tasks import fire_follow_triggers
if not settings.MAGE_AUTH_TOKEN: # pragma: no cover
return JsonResponse(dict(error="Authentication not configured"),
status=401)
authorization = request.META.get('HTTP_AUTHORIZATION', '').split(' ')
if len(authorization) != 2 or authorization[
0] != 'Token' or authorization[1] != settings.MAGE_AUTH_TOKEN:
return JsonResponse(dict(error="Incorrect authentication token"),
status=401)
action = kwargs['action'].lower()
new_contact = request.POST.get('new_contact',
'').lower() in ('true', '1')
if action == 'handle_message':
try:
msg_id = int(request.POST.get('message_id', ''))
except ValueError:
return JsonResponse(dict(error="Invalid message_id"),
status=400)
msg = Msg.objects.select_related('org').get(pk=msg_id)
push_task(
msg.org, HANDLER_QUEUE, HANDLE_EVENT_TASK,
dict(type=MSG_EVENT,
id=msg.id,
from_mage=True,
new_contact=new_contact))
# fire an event off for this message
WebHookEvent.trigger_sms_event(WebHookEvent.TYPE_SMS_RECEIVED, msg,
msg.created_on)
elif action == 'follow_notification':
try:
channel_id = int(request.POST.get('channel_id', ''))
contact_urn_id = int(request.POST.get('contact_urn_id', ''))
except ValueError: # pragma: needs cover
return JsonResponse(
dict(error="Invalid channel or contact URN id"),
status=400)
on_transaction_commit(lambda: fire_follow_triggers.apply_async(
args=(channel_id, contact_urn_id, new_contact),
queue='handler'))
elif action == 'stop_contact':
contact = Contact.objects.filter(is_active=True,
id=request.POST.get(
'contact_id', '-1')).first()
if not contact:
return JsonResponse(dict(error="Invalid contact_id"),
status=400)
contact.stop(contact.modified_by)
return JsonResponse(dict(error=None))
