def check_campaigns_task(sched_id=None): """ See if any event fires need to be triggered """ logger = check_campaigns_task.get_logger() # get a lock r = get_redis_connection() key = 'check_campaigns' # only do this if we aren't already checking campaigns if not r.get(key): with r.lock(key, timeout=3600): # for each that needs to be fired for fire in EventFire.objects.filter( fired=None, scheduled__lte=timezone.now()).select_related( 'contact', 'contact__org'): try: push_task(fire.contact.org, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, id=fire.id)) except Exception: # pragma: no cover logger.error("Error running campaign event: %s" % fire.pk, exc_info=True)
def send_msg_task(): """ Pops the next message off of our msg queue to send. """ # pop off the next task msg_tasks = pop_task(SEND_MSG_TASK) # it is possible we have no message to send, if so, just return if not msg_tasks: return if not isinstance(msg_tasks, list): msg_tasks = [msg_tasks] r = get_redis_connection() # acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time try: with r.lock('send_contact_%d' % msg_tasks[0]['contact'], timeout=300): # send each of our msgs while msg_tasks: msg_task = msg_tasks.pop(0) msg = dict_to_struct('MockMsg', msg_task, datetime_fields=['modified_on', 'sent_on', 'created_on', 'queued_on', 'next_attempt']) Channel.send_message(msg) # if there are more messages to send for this contact, sleep a second before moving on if msg_tasks: time.sleep(1) finally: # pragma: no cover # if some msgs weren't sent for some reason, then requeue them for later sending if msg_tasks: # requeue any unsent msgs push_task(msg_tasks[0]['org'], MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()) runs = runs.only('id', 'org', 'timeout_on') queued_timeouts = QueueRecord( 'flow_timeouts', lambda r: '%d:%d' % (r.id, datetime_to_epoch(r.timeout_on))) for run in runs: # ignore any run which was locked by previous calls to this task if not queued_timeouts.is_queued(run): try: task_payload = dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on) push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, task_payload) queued_timeouts.set_queued([run]) except Exception: # pragma: no cover logger.error("Error queuing timeout task for run #%d" % run.id, exc_info=True)
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ r = get_redis_connection() # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()) runs = runs.only('id', 'org', 'timeout_on') for run in runs: run_key = '%d:%d' % (run.id, datetime_to_epoch(run.timeout_on)) # check whether we have already queued this timeout pipe = r.pipeline() pipe.sismember(timezone.now().strftime(FLOW_TIMEOUT_KEY), run_key) pipe.sismember( (timezone.now() - timedelta(days=1)).strftime(FLOW_TIMEOUT_KEY), run_key) (queued_today, queued_yesterday) = pipe.execute() # if not, add a task to handle the timeout if not queued_today and not queued_yesterday: push_task( run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on)) # tag this run as being worked on so we don't double queue pipe = r.pipeline() sent_key = timezone.now().strftime(FLOW_TIMEOUT_KEY) pipe.sadd(sent_key, run_key) pipe.expire(sent_key, 86400) pipe.execute()
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()) runs = runs.only('id', 'org', 'timeout_on') for run in runs: # move this flow forward via the handler queue push_task( run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_campaigns_task(): """ See if any event fires need to be triggered """ # for each that needs to be fired for fire in EventFire.objects.filter( fired=None, scheduled__lte=timezone.now()).select_related( 'contact', 'contact__org'): try: push_task(fire.contact.org, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, id=fire.id)) except Exception: # pragma: no cover logger.error("Error running campaign event: %s" % fire.pk, exc_info=True)
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ r = get_redis_connection() # only do this if we aren't already expiring things key = 'check_flow_timeouts' if not r.get(key): with r.lock(key, timeout=900): # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()).only('id', 'org') for run in runs: # move this flow forward via the handler queue push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ r = get_redis_connection() # only do this if we aren't already expiring things key = 'check_flow_timeouts' if not r.get(key): with r.lock(key, timeout=900): # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()) runs = runs.only('id', 'org', 'timeout_on') for run in runs: # move this flow forward via the handler queue push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on))
def check_campaigns_task(): """ See if any event fires need to be triggered """ from temba.flows.models import Flow unfired = EventFire.objects.filter(fired=None, scheduled__lte=timezone.now()) unfired = unfired.values('id', 'event__flow_id') # group fire events by flow so they can be batched fire_ids_by_flow_id = defaultdict(list) for fire in unfired: fire_ids_by_flow_id[fire['event__flow_id']].append(fire['id']) # fetch the flows used by all these event fires flows_by_id = { flow.id: flow for flow in Flow.objects.filter(id__in=fire_ids_by_flow_id.keys()) } queued_fires = QueueRecord('queued_event_fires') # create queued tasks for flow_id, fire_ids in six.iteritems(fire_ids_by_flow_id): flow = flows_by_id[flow_id] # create sub-batches no no single task is too big for fire_id_batch in chunk_list(fire_ids, 500): # ignore any fires which were queued by previous calls to this task but haven't yet been marked as fired queued_fire_ids = queued_fires.filter_unqueued(fire_id_batch) if queued_fire_ids: try: push_task(flow.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, fires=queued_fire_ids)) queued_fires.set_queued(queued_fire_ids) except Exception: # pragma: no cover fire_ids_str = ','.join( six.text_type(f) for f in queued_fire_ids) logger.error("Error queuing campaign event fires: %s" % fire_ids_str, exc_info=True)
def check_campaigns_task(): """ See if any event fires need to be triggered """ from temba.flows.models import Flow unfired = EventFire.objects.filter( fired=None, scheduled__lte=timezone.now(), event__flow__flow_server_enabled=False ).select_related("event") unfired = unfired.values("id", "event_id", "event__flow_id") # group fire events by event so they can be batched fire_ids_by_event_id = defaultdict(list) event_flow_map = dict() for fire in unfired: event_flow_map[fire["event_id"]] = fire["event__flow_id"] fire_ids_by_event_id[fire["event_id"]].append(fire["id"]) # fetch the flows used by all these event fires flows_by_id = {flow.id: flow for flow in Flow.objects.filter(id__in=event_flow_map.values())} queued_fires = QueueRecord("queued_event_fires") # create queued tasks for ev_id, fire_ids in fire_ids_by_event_id.items(): flow_id = event_flow_map[ev_id] flow = flows_by_id[flow_id] # create sub-batches no no single task is too big for fire_id_batch in chunk_list(fire_ids, 500): # ignore any fires which were queued by previous calls to this task but haven't yet been marked as fired queued_fire_ids = queued_fires.filter_unqueued(fire_id_batch) if queued_fire_ids: try: push_task( flow.org_id, Queue.HANDLER, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, fires=queued_fire_ids) ) queued_fires.set_queued(queued_fire_ids) except Exception: # pragma: no cover fire_ids_str = ",".join(str(f) for f in queued_fire_ids) logger.error("Error queuing campaign event fires: %s" % fire_ids_str, exc_info=True)
def send_msg_task(): """ Pops the next message off of our msg queue to send. """ # pop off the next task org_id, msg_tasks = start_task(SEND_MSG_TASK) # it is possible we have no message to send, if so, just return if not msg_tasks: # pragma: needs cover return if not isinstance(msg_tasks, list): # pragma: needs cover msg_tasks = [msg_tasks] r = get_redis_connection() # acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time try: with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300): # send each of our msgs while msg_tasks: msg_task = msg_tasks.pop(0) msg = dict_to_struct( "MockMsg", msg_task, datetime_fields=[ "modified_on", "sent_on", "created_on", "queued_on", "next_attempt" ], ) Channel.send_message(msg) # if there are more messages to send for this contact, sleep a second before moving on if msg_tasks: time.sleep(1) finally: # pragma: no cover # mark this worker as done complete_task(SEND_MSG_TASK, org_id) # if some msgs weren't sent for some reason, then requeue them for later sending if msg_tasks: # requeue any unsent msgs push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_flow_timeouts_task(): """ See if any flow runs have timed out """ # find any runs that should have timed out runs = FlowRun.objects.filter(is_active=True, timeout_on__lte=timezone.now()) runs = runs.only("id", "org", "timeout_on") queued_timeouts = QueueRecord("flow_timeouts", lambda r: "%d:%d" % (r.id, datetime_to_epoch(r.timeout_on))) for run in runs: # ignore any run which was locked by previous calls to this task if not queued_timeouts.is_queued(run): try: task_payload = dict(type=TIMEOUT_EVENT, run=run.id, timeout_on=run.timeout_on) push_task(run.org_id, HANDLER_QUEUE, HANDLE_EVENT_TASK, task_payload) queued_timeouts.set_queued([run]) except Exception: # pragma: no cover logger.error("Error queuing timeout task for run #%d" % run.id, exc_info=True)
def send_msg_task(): """ Pops the next message off of our msg queue to send. """ # pop off the next task org_id, msg_tasks = start_task(SEND_MSG_TASK) # it is possible we have no message to send, if so, just return if not msg_tasks: # pragma: needs cover return if not isinstance(msg_tasks, list): # pragma: needs cover msg_tasks = [msg_tasks] r = get_redis_connection() # acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time try: with r.lock("send_contact_%d" % msg_tasks[0]["contact"], timeout=300): # send each of our msgs while msg_tasks: msg_task = msg_tasks.pop(0) msg = dict_to_struct( "MockMsg", msg_task, datetime_fields=["modified_on", "sent_on", "created_on", "queued_on", "next_attempt"], ) Channel.send_message(msg) # if there are more messages to send for this contact, sleep a second before moving on if msg_tasks: time.sleep(1) finally: # pragma: no cover # mark this worker as done complete_task(SEND_MSG_TASK, org_id) # if some msgs weren't sent for some reason, then requeue them for later sending if msg_tasks: # requeue any unsent msgs push_task(org_id, MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def check_campaigns_task(sched_id=None): """ See if any event fires need to be triggered """ logger = check_campaigns_task.get_logger() # get a lock r = get_redis_connection() key = 'check_campaigns' # only do this if we aren't already checking campaigns if not r.get(key): with r.lock(key, timeout=3600): # for each that needs to be fired for fire in EventFire.objects.filter(fired=None, scheduled__lte=timezone.now()).select_related('contact', 'contact.org'): try: push_task(fire.contact.org, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=FIRE_EVENT, id=fire.id)) except: # pragma: no cover logger.error("Error running campaign event: %s" % fire.pk, exc_info=True)
def post(self, request, *args, **kwargs): from temba.triggers.tasks import fire_follow_triggers if not settings.MAGE_AUTH_TOKEN: # pragma: no cover return JsonResponse(dict(error="Authentication not configured"), status=401) authorization = request.META.get('HTTP_AUTHORIZATION', '').split(' ') if len(authorization) != 2 or authorization[ 0] != 'Token' or authorization[1] != settings.MAGE_AUTH_TOKEN: return JsonResponse(dict(error="Incorrect authentication token"), status=401) action = kwargs['action'].lower() new_contact = request.POST.get('new_contact', '').lower() in ('true', '1') if action == 'handle_message': try: msg_id = int(request.POST.get('message_id', '')) except ValueError: return JsonResponse(dict(error="Invalid message_id"), status=400) msg = Msg.objects.select_related('org').get(pk=msg_id) push_task( msg.org, HANDLER_QUEUE, HANDLE_EVENT_TASK, dict(type=MSG_EVENT, id=msg.id, from_mage=True, new_contact=new_contact)) # fire an event off for this message WebHookEvent.trigger_sms_event(WebHookEvent.TYPE_SMS_RECEIVED, msg, msg.created_on) elif action == 'follow_notification': try: channel_id = int(request.POST.get('channel_id', '')) contact_urn_id = int(request.POST.get('contact_urn_id', '')) except ValueError: # pragma: needs cover return JsonResponse( dict(error="Invalid channel or contact URN id"), status=400) on_transaction_commit(lambda: fire_follow_triggers.apply_async( args=(channel_id, contact_urn_id, new_contact), queue='handler')) elif action == 'stop_contact': contact = Contact.objects.filter(is_active=True, id=request.POST.get( 'contact_id', '-1')).first() if not contact: return JsonResponse(dict(error="Invalid contact_id"), status=400) contact.stop(contact.modified_by) return JsonResponse(dict(error=None))