def handle_event_task(): """ Priority queue task that handles both event fires (when fired) and new incoming messages that need to be handled. Currently three types of events may be "popped" from our queue: msg - Which contains the id of the Msg to be processed fire - Which contains the id of the EventFire that needs to be fired timeout - Which contains a run that timed out and needs to be resumed """ # pop off the next task org_id, event_task = start_task(HANDLE_EVENT_TASK) # it is possible we have no message to send, if so, just return if not event_task: # pragma: needs cover return try: if event_task['type'] == MSG_EVENT: process_message_task(event_task) elif event_task['type'] == FIRE_EVENT: fire_ids = event_task.get('fires') if 'fires' in event_task else [ event_task.get('id') ] process_fire_events(fire_ids) elif event_task['type'] == TIMEOUT_EVENT: timeout_on = json_date_to_datetime(event_task['timeout_on']) process_run_timeout(event_task['run'], timeout_on) else: # pragma: needs cover raise Exception("Unexpected event type: %s" % event_task) finally: complete_task(HANDLE_EVENT_TASK, org_id)
def start_msg_flow_batch_task(): # pop off the next task org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH) # it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so if task_obj is None: # pragma: needs cover return start = time.time() try: # instantiate all the objects we need that were serialized as JSON flow = Flow.objects.filter(pk=task_obj['flow'], is_active=True).first() if not flow: # pragma: needs cover return broadcasts = [] if not task_obj['broadcasts'] else Broadcast.objects.filter(pk__in=task_obj['broadcasts']) started_flows = [] if not task_obj['started_flows'] else task_obj['started_flows'] start_msg = None if not task_obj['start_msg'] else Msg.objects.filter(id=task_obj['start_msg']).first() extra = task_obj['extra'] flow_start = None if not task_obj['flow_start'] else FlowStart.objects.filter(pk=task_obj['flow_start']).first() contacts = task_obj['contacts'] # and go do our work flow.start_msg_flow_batch(contacts, broadcasts=broadcasts, started_flows=started_flows, start_msg=start_msg, extra=extra, flow_start=flow_start) finally: complete_task(Flow.START_MSG_FLOW_BATCH, org_id) print("Started batch of %d contacts in flow %d [%d] in %0.3fs" % (len(contacts), flow.id, flow.org_id, time.time() - start))
def start_msg_flow_batch_task(): # pop off the next task org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH) # it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so if task_obj is None: # pragma: needs cover return start = time.time() try: task_type = task_obj.get("task_type", FLOW_BATCH) if task_type == FLOW_BATCH: # instantiate all the objects we need that were serialized as JSON flow = Flow.objects.filter(pk=task_obj["flow"], is_active=True, is_archived=False).first() if not flow: # pragma: needs cover return broadcasts = [] if not task_obj[ "broadcasts"] else Broadcast.objects.filter( pk__in=task_obj["broadcasts"]) started_flows = [] if not task_obj["started_flows"] else task_obj[ "started_flows"] start_msg = None if not task_obj[ "start_msg"] else Msg.objects.filter( id=task_obj["start_msg"]).first() extra = task_obj["extra"] flow_start = (None if not task_obj["flow_start"] else FlowStart.objects.filter( pk=task_obj["flow_start"]).first()) contacts = task_obj["contacts"] # and go do our work flow.start_msg_flow_batch( contacts, broadcasts=broadcasts, started_flows=started_flows, start_msg=start_msg, extra=extra, flow_start=flow_start, ) print("Started batch of %d contacts in flow %d [%d] in %0.3fs" % (len(contacts), flow.id, flow.org_id, time.time() - start)) elif task_type == BROADCAST_BATCH: broadcast = Broadcast.objects.filter( org_id=org_id, id=task_obj["broadcast"]).first() if broadcast: broadcast.send_batch(**task_obj["kwargs"]) print("Sent batch of %d messages in broadcast [%d] in %0.3fs" % (len(task_obj["kwargs"]["urn_ids"]), broadcast.id, time.time() - start)) finally: complete_task(Flow.START_MSG_FLOW_BATCH, org_id)
def start_msg_flow_batch_task(): # pop off the next task org_id, task_obj = start_task(Flow.START_MSG_FLOW_BATCH) # it is possible that somehow we might get None back if more workers were started than tasks got added, bail if so if task_obj is None: # pragma: needs cover return start = time.time() try: task_type = task_obj.get("task_type", FLOW_BATCH) if task_type == FLOW_BATCH: # instantiate all the objects we need that were serialized as JSON flow = Flow.objects.filter(pk=task_obj["flow"], is_active=True, is_archived=False).first() if not flow: # pragma: needs cover return broadcasts = [] if not task_obj["broadcasts"] else Broadcast.objects.filter(pk__in=task_obj["broadcasts"]) started_flows = [] if not task_obj["started_flows"] else task_obj["started_flows"] start_msg = None if not task_obj["start_msg"] else Msg.objects.filter(id=task_obj["start_msg"]).first() extra = task_obj["extra"] flow_start = ( None if not task_obj["flow_start"] else FlowStart.objects.filter(pk=task_obj["flow_start"]).first() ) contacts = task_obj["contacts"] # and go do our work flow.start_msg_flow_batch( contacts, broadcasts=broadcasts, started_flows=started_flows, start_msg=start_msg, extra=extra, flow_start=flow_start, ) print( "Started batch of %d contacts in flow %d [%d] in %0.3fs" % (len(contacts), flow.id, flow.org_id, time.time() - start) ) elif task_type == BROADCAST_BATCH: broadcast = Broadcast.objects.filter(org_id=org_id, id=task_obj["broadcast"]).first() if broadcast: broadcast.send_batch(**task_obj["kwargs"]) print( "Sent batch of %d messages in broadcast [%d] in %0.3fs" % (len(task_obj["kwargs"]["urn_ids"]), broadcast.id, time.time() - start) ) finally: complete_task(Flow.START_MSG_FLOW_BATCH, org_id)
def handle_event_task(): """ Priority queue task that handles both event fires (when fired) and new incoming messages that need to be handled. Currently three types of events may be "popped" from our queue: msg - Which contains the id of the Msg to be processed fire - Which contains the id of the EventFire that needs to be fired timeout - Which contains a run that timed out and needs to be resumed """ from temba.campaigns.models import EventFire r = get_redis_connection() # pop off the next task org_id, event_task = start_task(HANDLE_EVENT_TASK) # it is possible we have no message to send, if so, just return if not event_task: # pragma: needs cover return try: if event_task['type'] == MSG_EVENT: process_message_task(event_task['id'], event_task.get('from_mage', False), event_task.get('new_contact', False)) elif event_task['type'] == FIRE_EVENT: # use a lock to make sure we don't do two at once somehow key = 'fire_campaign_%s' % event_task['id'] if not r.get(key): with r.lock(key, timeout=120): event = EventFire.objects.filter(pk=event_task['id'], fired=None)\ .select_related('event', 'event__campaign', 'event__campaign__org').first() if event: print "E[%09d] Firing for org: %s" % ( event.id, event.event.campaign.org.name) start = time.time() event.fire() print "E[%09d] %08.3f s" % (event.id, time.time() - start) elif event_task['type'] == TIMEOUT_EVENT: timeout_on = json_date_to_datetime(event_task['timeout_on']) process_run_timeout(event_task['run'], timeout_on) else: # pragma: needs cover raise Exception("Unexpected event type: %s" % event_task) finally: complete_task(HANDLE_EVENT_TASK, org_id)
def send_msg_task(): """ Pops the next message off of our msg queue to send. """ # pop off the next task org_id, msg_tasks = start_task(SEND_MSG_TASK) # it is possible we have no message to send, if so, just return if not msg_tasks: # pragma: needs cover return if not isinstance(msg_tasks, list): # pragma: needs cover msg_tasks = [msg_tasks] r = get_redis_connection() # acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time try: with r.lock('send_contact_%d' % msg_tasks[0]['contact'], timeout=300): # send each of our msgs while msg_tasks: msg_task = msg_tasks.pop(0) msg = dict_to_struct('MockMsg', msg_task, datetime_fields=[ 'modified_on', 'sent_on', 'created_on', 'queued_on', 'next_attempt' ]) # we renamed msg.session_id to msg.connection_id but might still have queued messages with the former if hasattr(msg, 'session_id'): msg.connection_id = msg.session_id Channel.send_message(msg) # if there are more messages to send for this contact, sleep a second before moving on if msg_tasks: time.sleep(1) finally: # pragma: no cover # mark this worker as done complete_task(SEND_MSG_TASK, org_id) # if some msgs weren't sent for some reason, then requeue them for later sending if msg_tasks: # requeue any unsent msgs push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def handle_event_task(): """ Priority queue task that handles both event fires (when fired) and new incoming messages that need to be handled. Currently three types of events may be "popped" from our queue: msg - Which contains the id of the Msg to be processed fire - Which contains the id of the EventFire that needs to be fired timeout - Which contains a run that timed out and needs to be resumed stop_contact - Which contains the contact id to stop """ # pop off the next task org_id, event_task = start_task(HANDLE_EVENT_TASK) # it is possible we have no message to send, if so, just return if not event_task: # pragma: needs cover return try: if event_task["type"] == MSG_EVENT: process_message_task(event_task) elif event_task["type"] == FIRE_EVENT: fire_ids = event_task.get("fires") if "fires" in event_task else [ event_task.get("id") ] process_fire_events(fire_ids) elif event_task["type"] == TIMEOUT_EVENT: timeout_on = iso8601.parse_date(event_task["timeout_on"]) process_run_timeout(event_task["run"], timeout_on) elif event_task["type"] == STOP_CONTACT_EVENT: contact = Contact.objects.get(id=event_task["contact_id"]) contact.stop(contact.modified_by) elif event_task["type"] == CHANNEL_EVENT: event = ChannelEvent.objects.get(id=event_task["event_id"]) event.handle() else: # pragma: needs cover raise Exception("Unexpected event type: %s" % event_task) finally: complete_task(HANDLE_EVENT_TASK, org_id)
def handle_event_task(): """ Priority queue task that handles both event fires (when fired) and new incoming messages that need to be handled. Currently three types of events may be "popped" from our queue: msg - Which contains the id of the Msg to be processed fire - Which contains the id of the EventFire that needs to be fired timeout - Which contains a run that timed out and needs to be resumed stop_contact - Which contains the contact id to stop """ # pop off the next task org_id, event_task = start_task(HANDLE_EVENT_TASK) # it is possible we have no message to send, if so, just return if not event_task: # pragma: needs cover return try: if event_task["type"] == MSG_EVENT: process_message_task(event_task) elif event_task["type"] == FIRE_EVENT: fire_ids = event_task.get("fires") if "fires" in event_task else [event_task.get("id")] process_fire_events(fire_ids) elif event_task["type"] == TIMEOUT_EVENT: timeout_on = iso8601.parse_date(event_task["timeout_on"]) process_run_timeout(event_task["run"], timeout_on) elif event_task["type"] == STOP_CONTACT_EVENT: contact = Contact.objects.get(id=event_task["contact_id"]) contact.stop(contact.modified_by) elif event_task["type"] == CHANNEL_EVENT: event = ChannelEvent.objects.get(id=event_task["event_id"]) event.handle() else: # pragma: needs cover raise Exception("Unexpected event type: %s" % event_task) finally: complete_task(HANDLE_EVENT_TASK, org_id)
def send_msg_task(): """ Pops the next message off of our msg queue to send. """ # pop off the next task org_id, msg_tasks = start_task(SEND_MSG_TASK) # it is possible we have no message to send, if so, just return if not msg_tasks: # pragma: needs cover return if not isinstance(msg_tasks, list): # pragma: needs cover msg_tasks = [msg_tasks] r = get_redis_connection() # acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time try: with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300): # send each of our msgs while msg_tasks: msg_task = msg_tasks.pop(0) msg = dict_to_struct( "MockMsg", msg_task, datetime_fields=["modified_on", "sent_on", "created_on", "queued_on", "next_attempt"], ) Channel.send_message(msg) # if there are more messages to send for this contact, sleep a second before moving on if msg_tasks: time.sleep(1) finally: # pragma: no cover # mark this worker as done complete_task(SEND_MSG_TASK, org_id) # if some msgs weren't sent for some reason, then requeue them for later sending if msg_tasks: # requeue any unsent msgs push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)