def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently three types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
timeout - Which contains a run that timed out and needs to be resumed
"""
# pop off the next task
org_id, event_task = start_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task: # pragma: needs cover
return
try:
if event_task['type'] == MSG_EVENT:
process_message_task(event_task)
elif event_task['type'] == FIRE_EVENT:
fire_ids = event_task.get('fires') if 'fires' in event_task else [
event_task.get('id')
]
process_fire_events(fire_ids)
elif event_task['type'] == TIMEOUT_EVENT:
timeout_on = json_date_to_datetime(event_task['timeout_on'])
process_run_timeout(event_task['run'], timeout_on)
else: # pragma: needs cover
raise Exception("Unexpected event type: %s" % event_task)
finally:
complete_task(HANDLE_EVENT_TASK, org_id)
def start_msg_flow_batch_task():
# pop off the next task
org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH)
# it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so
if task_obj is None: # pragma: needs cover
return
start = time.time()
try:
# instantiate all the objects we need that were serialized as JSON
flow = Flow.objects.filter(pk=task_obj['flow'], is_active=True).first()
if not flow: # pragma: needs cover
return
broadcasts = [] if not task_obj['broadcasts'] else Broadcast.objects.filter(pk__in=task_obj['broadcasts'])
started_flows = [] if not task_obj['started_flows'] else task_obj['started_flows']
start_msg = None if not task_obj['start_msg'] else Msg.objects.filter(id=task_obj['start_msg']).first()
extra = task_obj['extra']
flow_start = None if not task_obj['flow_start'] else FlowStart.objects.filter(pk=task_obj['flow_start']).first()
contacts = task_obj['contacts']
# and go do our work
flow.start_msg_flow_batch(contacts, broadcasts=broadcasts,
started_flows=started_flows, start_msg=start_msg,
extra=extra, flow_start=flow_start)
finally:
complete_task(Flow.START_MSG_FLOW_BATCH, org_id)
print("Started batch of %d contacts in flow %d [%d] in %0.3fs"
% (len(contacts), flow.id, flow.org_id, time.time() - start))
def start_msg_flow_batch_task():
# pop off the next task
org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH)
# it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so
if task_obj is None: # pragma: needs cover
return
start = time.time()
try:
task_type = task_obj.get("task_type", FLOW_BATCH)
if task_type == FLOW_BATCH:
# instantiate all the objects we need that were serialized as JSON
flow = Flow.objects.filter(pk=task_obj["flow"],
is_active=True,
is_archived=False).first()
if not flow: # pragma: needs cover
return
broadcasts = [] if not task_obj[
"broadcasts"] else Broadcast.objects.filter(
pk__in=task_obj["broadcasts"])
started_flows = [] if not task_obj["started_flows"] else task_obj[
"started_flows"]
start_msg = None if not task_obj[
"start_msg"] else Msg.objects.filter(
id=task_obj["start_msg"]).first()
extra = task_obj["extra"]
flow_start = (None if not task_obj["flow_start"] else
FlowStart.objects.filter(
pk=task_obj["flow_start"]).first())
contacts = task_obj["contacts"]
# and go do our work
flow.start_msg_flow_batch(
contacts,
broadcasts=broadcasts,
started_flows=started_flows,
start_msg=start_msg,
extra=extra,
flow_start=flow_start,
)
print("Started batch of %d contacts in flow %d [%d] in %0.3fs" %
(len(contacts), flow.id, flow.org_id, time.time() - start))
elif task_type == BROADCAST_BATCH:
broadcast = Broadcast.objects.filter(
org_id=org_id, id=task_obj["broadcast"]).first()
if broadcast:
broadcast.send_batch(**task_obj["kwargs"])
print("Sent batch of %d messages in broadcast [%d] in %0.3fs" %
(len(task_obj["kwargs"]["urn_ids"]), broadcast.id,
time.time() - start))
finally:
complete_task(Flow.START_MSG_FLOW_BATCH, org_id)
def start_msg_flow_batch_task():
# pop off the next task
org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH)
# it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so
if task_obj is None: # pragma: needs cover
return
start = time.time()
try:
task_type = task_obj.get("task_type", FLOW_BATCH)
if task_type == FLOW_BATCH:
# instantiate all the objects we need that were serialized as JSON
flow = Flow.objects.filter(pk=task_obj["flow"], is_active=True, is_archived=False).first()
if not flow: # pragma: needs cover
return
broadcasts = [] if not task_obj["broadcasts"] else Broadcast.objects.filter(pk__in=task_obj["broadcasts"])
started_flows = [] if not task_obj["started_flows"] else task_obj["started_flows"]
start_msg = None if not task_obj["start_msg"] else Msg.objects.filter(id=task_obj["start_msg"]).first()
extra = task_obj["extra"]
flow_start = (
None if not task_obj["flow_start"] else FlowStart.objects.filter(pk=task_obj["flow_start"]).first()
)
contacts = task_obj["contacts"]
# and go do our work
flow.start_msg_flow_batch(
contacts,
broadcasts=broadcasts,
started_flows=started_flows,
start_msg=start_msg,
extra=extra,
flow_start=flow_start,
)
print(
"Started batch of %d contacts in flow %d [%d] in %0.3fs"
% (len(contacts), flow.id, flow.org_id, time.time() - start)
)
elif task_type == BROADCAST_BATCH:
broadcast = Broadcast.objects.filter(org_id=org_id, id=task_obj["broadcast"]).first()
if broadcast:
broadcast.send_batch(**task_obj["kwargs"])
print(
"Sent batch of %d messages in broadcast [%d] in %0.3fs"
% (len(task_obj["kwargs"]["urn_ids"]), broadcast.id, time.time() - start)
)
finally:
complete_task(Flow.START_MSG_FLOW_BATCH, org_id)
def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently three types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
timeout - Which contains a run that timed out and needs to be resumed
"""
from temba.campaigns.models import EventFire
r = get_redis_connection()
# pop off the next task
org_id, event_task = start_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task: # pragma: needs cover
return
try:
if event_task['type'] == MSG_EVENT:
process_message_task(event_task['id'],
event_task.get('from_mage', False),
event_task.get('new_contact', False))
elif event_task['type'] == FIRE_EVENT:
# use a lock to make sure we don't do two at once somehow
key = 'fire_campaign_%s' % event_task['id']
if not r.get(key):
with r.lock(key, timeout=120):
event = EventFire.objects.filter(pk=event_task['id'], fired=None)\
.select_related('event', 'event__campaign', 'event__campaign__org').first()
if event:
print "E[%09d] Firing for org: %s" % (
event.id, event.event.campaign.org.name)
start = time.time()
event.fire()
print "E[%09d] %08.3f s" % (event.id,
time.time() - start)
elif event_task['type'] == TIMEOUT_EVENT:
timeout_on = json_date_to_datetime(event_task['timeout_on'])
process_run_timeout(event_task['run'], timeout_on)
else: # pragma: needs cover
raise Exception("Unexpected event type: %s" % event_task)
finally:
complete_task(HANDLE_EVENT_TASK, org_id)
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
org_id, msg_tasks = start_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks: # pragma: needs cover
return
if not isinstance(msg_tasks, list): # pragma: needs cover
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock('send_contact_%d' % msg_tasks[0]['contact'], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct('MockMsg',
msg_task,
datetime_fields=[
'modified_on', 'sent_on',
'created_on', 'queued_on',
'next_attempt'
])
# we renamed msg.session_id to msg.connection_id but might still have queued messages with the former
if hasattr(msg, 'session_id'):
msg.connection_id = msg.session_id
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# mark this worker as done
complete_task(SEND_MSG_TASK, org_id)
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently three types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
timeout - Which contains a run that timed out and needs to be resumed
stop_contact - Which contains the contact id to stop
"""
# pop off the next task
org_id, event_task = start_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task: # pragma: needs cover
return
try:
if event_task["type"] == MSG_EVENT:
process_message_task(event_task)
elif event_task["type"] == FIRE_EVENT:
fire_ids = event_task.get("fires") if "fires" in event_task else [
event_task.get("id")
]
process_fire_events(fire_ids)
elif event_task["type"] == TIMEOUT_EVENT:
timeout_on = iso8601.parse_date(event_task["timeout_on"])
process_run_timeout(event_task["run"], timeout_on)
elif event_task["type"] == STOP_CONTACT_EVENT:
contact = Contact.objects.get(id=event_task["contact_id"])
contact.stop(contact.modified_by)
elif event_task["type"] == CHANNEL_EVENT:
event = ChannelEvent.objects.get(id=event_task["event_id"])
event.handle()
else: # pragma: needs cover
raise Exception("Unexpected event type: %s" % event_task)
finally:
complete_task(HANDLE_EVENT_TASK, org_id)
def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently three types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
timeout - Which contains a run that timed out and needs to be resumed
stop_contact - Which contains the contact id to stop
"""
# pop off the next task
org_id, event_task = start_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task: # pragma: needs cover
return
try:
if event_task["type"] == MSG_EVENT:
process_message_task(event_task)
elif event_task["type"] == FIRE_EVENT:
fire_ids = event_task.get("fires") if "fires" in event_task else [event_task.get("id")]
process_fire_events(fire_ids)
elif event_task["type"] == TIMEOUT_EVENT:
timeout_on = iso8601.parse_date(event_task["timeout_on"])
process_run_timeout(event_task["run"], timeout_on)
elif event_task["type"] == STOP_CONTACT_EVENT:
contact = Contact.objects.get(id=event_task["contact_id"])
contact.stop(contact.modified_by)
elif event_task["type"] == CHANNEL_EVENT:
event = ChannelEvent.objects.get(id=event_task["event_id"])
event.handle()
else: # pragma: needs cover
raise Exception("Unexpected event type: %s" % event_task)
finally:
complete_task(HANDLE_EVENT_TASK, org_id)
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
org_id, msg_tasks = start_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks: # pragma: needs cover
return
if not isinstance(msg_tasks, list): # pragma: needs cover
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct(
"MockMsg",
msg_task,
datetime_fields=["modified_on", "sent_on", "created_on", "queued_on", "next_attempt"],
)
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# mark this worker as done
complete_task(SEND_MSG_TASK, org_id)
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
