def msg_as_task(msg): """ Used to serialize msgs as tasks to courier """ msg_json = dict( id=msg.id, uuid=str(msg.uuid) if msg.uuid else "", org_id=msg.org_id, channel_id=msg.channel_id, channel_uuid=msg.channel.uuid, contact_id=msg.contact_id, contact_urn_id=msg.contact_urn_id, status=msg.status, direction=msg.direction, text=msg.text, high_priority=msg.high_priority, priority=500 if msg.high_priority else 100, # TODO stop using this on courier side and then remove urn=msg.contact_urn.urn, error_count=msg.error_count, attachments=msg.attachments, response_to_id=msg.response_to_id, external_id=msg.external_id, tps_cost=msg.channel.calculate_tps_cost(msg), next_attempt=datetime_to_str(msg.next_attempt, ms=True), created_on=datetime_to_str(msg.created_on, ms=True), modified_on=datetime_to_str(msg.modified_on, ms=True), queued_on=datetime_to_str(msg.queued_on, ms=True), sent_on=datetime_to_str(msg.sent_on, ms=True)) if msg.contact_urn.auth: # pragma: no cover msg_json['contact_urn_auth'] = msg.contact_urn.auth return msg_json
def msg_as_task(msg): """ Used to serialize msgs as tasks to courier """ msg_json = dict(id=msg.id, uuid=unicode(msg.uuid) if msg.uuid else "", org_id=msg.org_id, channel_id=msg.channel_id, channel_uuid=msg.channel.uuid, contact_id=msg.contact_id, contact_urn_id=msg.contact_urn_id, status=msg.status, direction=msg.direction, text=msg.text, priority=msg.priority, urn=msg.contact_urn.urn, error_count=msg.error_count, attachments=msg.attachments, response_to_id=msg.response_to_id, external_id=msg.external_id, next_attempt=datetime_to_str(msg.next_attempt, ms=True), created_on=datetime_to_str(msg.created_on, ms=True), modified_on=datetime_to_str(msg.modified_on, ms=True), queued_on=datetime_to_str(msg.queued_on, ms=True), sent_on=datetime_to_str(msg.sent_on, ms=True)) if msg.contact_urn.auth: # pragma: no cover msg_json['contact_urn_auth'] = msg.contact_urn.auth return msg_json
def _create_contact_batch(self, batch): """ Bulk creates a batch of contacts from flat representations """ for c in batch: c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'], is_stopped=c['is_stopped'], is_blocked=c['is_blocked'], is_active=c['is_active'], created_by=c['user'], created_on=c['created_on'], modified_by=c['user'], modified_on=c['modified_on']) Contact.objects.bulk_create([c['object'] for c in batch]) # now that contacts have pks, bulk create the actual URN, value and group membership objects batch_urns = [] batch_values = [] batch_memberships = [] for c in batch: org = c['org'] c['urns'] = [] if c['tel']: c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME, path=c['tel'], urn=URN.from_tel(c['tel']))) if c['twitter']: c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME, path=c['twitter'], urn=URN.from_twitter(c['twitter']))) if c['gender']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'], string_value=c['gender'])) if c['age']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'], string_value=str(c['age']), decimal_value=c['age'])) if c['joined']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'], string_value=datetime_to_str(c['joined']), datetime_value=c['joined'])) if c['ward']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'], string_value=c['ward'].name, location_value=c['ward'])) if c['district']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'], string_value=c['district'].name, location_value=c['district'])) if c['state']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'], string_value=c['state'].name, location_value=c['state'])) for g in c['groups']: batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g)) batch_urns += c['urns'] ContactURN.objects.bulk_create(batch_urns) Value.objects.bulk_create(batch_values) ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
# allow this much absolute change from previous results (milliseconds) ALLOWED_CHANGE_MAXIMUM = 50 # allow this much percentage change from previous results ALLOWED_CHANGE_PERCENTAGE = 5 # a org specific context used in URL generation URL_CONTEXT_TEMPLATE = { 'first-group': lambda org: ContactGroup.user_groups.filter(org=org).order_by('id').first( ).uuid, 'last-group': lambda org: ContactGroup.user_groups.filter(org=org).order_by('-id').first( ).uuid, '1-year-ago': lambda org: datetime_to_str(now() - timedelta(days=365), get_datetime_format(org.get_dayfirst())[0]) } TEST_URLS = ( '/api/v2/channels.json', '/api/v2/channel_events.json', '/api/v2/contacts.json', '/api/v2/contacts.json?deleted=true', '/api/v2/contacts.json?group={first-group}', '/api/v2/groups.json', '/api/v2/fields.json', '/api/v2/labels.json', '/api/v2/messages.json?folder=incoming', '/api/v2/messages.json?folder=inbox', '/api/v2/messages.json?folder=flows', '/api/v2/messages.json?folder=archived',
def trigger_flow_event(cls, webhook_url, flow, run, node_uuid, contact, event, action='POST', resthook=None): org = flow.org api_user = get_api_user() json_time = datetime_to_str(timezone.now()) # get the results for this contact results = flow.get_results(contact) values = [] if results and results[0]: values = results[0]['values'] for value in values: value['time'] = datetime_to_str(value['time']) value['value'] = unicode(value['value']) # if the action is on the first node # we might not have an sms (or channel) yet channel = None text = None contact_urn = contact.get_urn() if event: text = event.text channel = event.channel contact_urn = event.contact_urn if channel: channel_id = channel.pk else: channel_id = -1 steps = [] for step in run.steps.prefetch_related( 'messages', 'broadcasts').order_by('arrived_on'): steps.append( dict(type=step.step_type, node=step.step_uuid, arrived_on=datetime_to_str(step.arrived_on), left_on=datetime_to_str(step.left_on), text=step.get_text(), value=step.rule_value)) data = dict(channel=channel_id, relayer=channel_id, flow=flow.id, flow_name=flow.name, flow_base_language=flow.base_language, run=run.id, text=text, step=unicode(node_uuid), phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, formatted=False), contact=contact.uuid, urn=unicode(contact_urn), values=json.dumps(values), steps=json.dumps(steps), time=json_time) if not action: action = 'POST' webhook_event = WebHookEvent.objects.create(org=org, event=FLOW, channel=channel, data=json.dumps(data), try_count=1, action=action, resthook=resthook, created_by=api_user, modified_by=api_user) status_code = -1 message = "None" body = None # webhook events fire immediately since we need the results back try: # only send webhooks when we are configured to, otherwise fail if not settings.SEND_WEBHOOKS: raise Exception( "!! Skipping WebHook send, SEND_WEBHOOKS set to False") # no url, bail! if not webhook_url: raise Exception("No webhook_url specified, skipping send") # some hosts deny generic user agents, use Temba as our user agent if action == 'GET': response = requests.get(webhook_url, headers=TEMBA_HEADERS, timeout=10) else: response = requests.post(webhook_url, data=data, headers=TEMBA_HEADERS, timeout=10) response_text = response.text body = response.text status_code = response.status_code if response.status_code == 200 or response.status_code == 201: try: response_json = json.loads(response_text) # only update if we got a valid JSON dictionary or list if not isinstance(response_json, dict) and not isinstance( response_json, list): raise ValueError( "Response must be a JSON dictionary or list, ignoring response." ) run.update_fields(response_json) message = "Webhook called successfully." except ValueError as e: message = "Response must be a JSON dictionary, ignoring response." webhook_event.status = COMPLETE else: webhook_event.status = FAILED message = "Got non 200 response (%d) from webhook." % response.status_code raise Exception("Got non 200 response (%d) from webhook." % response.status_code) except Exception as e: import traceback traceback.print_exc() webhook_event.status = FAILED message = "Error calling webhook: %s" % unicode(e) finally: webhook_event.save() # make sure our message isn't too long if message: message = message[:255] result = WebHookResult.objects.create(event=webhook_event, url=webhook_url, status_code=status_code, body=body, message=message, data=urlencode(data, doseq=True), created_by=api_user, modified_by=api_user) # if this is a test contact, add an entry to our action log if run.contact.is_test: from temba.flows.models import ActionLog log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % ( reverse('api.log_read', args=[webhook_event.pk ]), status_code) ActionLog.create(run, log_txt, safe=True) return result
def trigger_flow_event(cls, webhook_url, flow, run, node, contact, event, action="POST"): org = flow.org api_user = get_api_user() # no-op if no webhook configured if not webhook_url: return json_time = datetime_to_str(timezone.now()) # get the results for this contact results = flow.get_results(contact) values = [] if results and results[0]: values = results[0]["values"] for value in values: value["time"] = datetime_to_str(value["time"]) value["value"] = unicode(value["value"]) # if the action is on the first node # we might not have an sms (or channel) yet channel = None text = None if event: text = event.text channel = event.channel if channel: channel_id = channel.pk else: channel_id = -1 steps = [] for step in run.steps.all().order_by("arrived_on"): steps.append( dict( type=step.step_type, node=step.step_uuid, arrived_on=datetime_to_str(step.arrived_on), left_on=datetime_to_str(step.left_on), text=step.get_text(), value=step.rule_value, ) ) data = dict( channel=channel_id, relayer=channel_id, flow=flow.id, run=run.id, text=text, step=unicode(node.uuid), phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, full=True), values=json.dumps(values), steps=json.dumps(steps), time=json_time, ) if not action: action = "POST" webhook_event = WebHookEvent.objects.create( org=org, event=FLOW, channel=channel, data=json.dumps(data), try_count=1, action=action, created_by=api_user, modified_by=api_user, ) status_code = -1 message = "None" body = None # webhook events fire immediately since we need the results back try: # only send webhooks when we are configured to, otherwise fail if not settings.SEND_WEBHOOKS: raise Exception("!! Skipping WebHook send, SEND_WEBHOOKS set to False") # some hosts deny generic user agents, use Temba as our user agent if action == "GET": response = requests.get(webhook_url, headers=TEMBA_HEADERS, timeout=10) else: response = requests.post(webhook_url, data=data, headers=TEMBA_HEADERS, timeout=10) response_text = response.text body = response.text status_code = response.status_code if response.status_code == 200 or response.status_code == 201: try: response_json = json.loads(response_text) # only update if we got a valid JSON dictionary if not isinstance(response_json, dict): raise ValueError("Response must be a JSON dictionary, ignoring response.") run.update_fields(response_json) message = "Webhook called successfully." except ValueError as e: message = "Response must be a JSON dictionary, ignoring response." webhook_event.status = COMPLETE else: webhook_event.status = FAILED message = "Got non 200 response (%d) from webhook." % response.status_code raise Exception("Got non 200 response (%d) from webhook." % response.status_code) except Exception as e: import traceback traceback.print_exc() webhook_event.status = FAILED message = "Error calling webhook: %s" % unicode(e) finally: webhook_event.save() result = WebHookResult.objects.create( event=webhook_event, url=webhook_url, status_code=status_code, body=body, message=message, data=urlencode(data, doseq=True), created_by=api_user, modified_by=api_user, ) # if this is a test contact, add an entry to our action log if run.contact.is_test: from temba.flows.models import ActionLog log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % ( reverse("api.log_read", args=[webhook_event.pk]), status_code, ) ActionLog.create(run, log_txt, safe=True) return result
def trigger_flow_event(cls, run, webhook_url, node_uuid, msg, action='POST', resthook=None, header=None): flow = run.flow org = flow.org contact = run.contact api_user = get_api_user() json_time = datetime_to_str(timezone.now()) # get the results for this contact results = run.flow.get_results(run.contact) values = [] if results and results[0]: values = results[0]['values'] for value in values: value['time'] = datetime_to_str(value['time']) value['value'] = six.text_type(value['value']) if msg: text = msg.text attachments = msg.get_attachments() channel = msg.channel contact_urn = msg.contact_urn else: # if the action is on the first node we might not have an sms (or channel) yet channel = None text = None attachments = [] contact_urn = contact.get_urn() steps = [] for step in run.steps.prefetch_related('messages', 'broadcasts').order_by('arrived_on'): steps.append(dict(type=step.step_type, node=step.step_uuid, arrived_on=datetime_to_str(step.arrived_on), left_on=datetime_to_str(step.left_on), text=step.get_text(), value=step.rule_value)) data = dict(channel=channel.id if channel else -1, channel_uuid=channel.uuid if channel else None, relayer=channel.id if channel else -1, flow=flow.id, flow_uuid=flow.uuid, flow_name=flow.name, flow_base_language=flow.base_language, run=run.id, text=text, attachments=[a.url for a in attachments], step=six.text_type(node_uuid), phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, formatted=False), contact=contact.uuid, contact_name=contact.name, urn=six.text_type(contact_urn), values=json.dumps(values), steps=json.dumps(steps), time=json_time, header=header) if not action: # pragma: needs cover action = 'POST' webhook_event = cls.objects.create(org=org, event=cls.TYPE_FLOW, channel=channel, data=json.dumps(data), run=run, try_count=1, action=action, resthook=resthook, created_by=api_user, modified_by=api_user) status_code = -1 message = "None" body = None start = time.time() # webhook events fire immediately since we need the results back try: # no url, bail! if not webhook_url: raise Exception("No webhook_url specified, skipping send") # only send webhooks when we are configured to, otherwise fail if settings.SEND_WEBHOOKS: requests_headers = TEMBA_HEADERS if header: requests_headers.update(header) # some hosts deny generic user agents, use Temba as our user agent if action == 'GET': response = requests.get(webhook_url, headers=requests_headers, timeout=10) else: response = requests.post(webhook_url, data=data, headers=requests_headers, timeout=10) body = response.text if body: body = body.strip() status_code = response.status_code else: print("!! Skipping WebHook send, SEND_WEBHOOKS set to False") body = 'Skipped actual send' status_code = 200 # process the webhook response try: response_json = json.loads(body, object_pairs_hook=OrderedDict) # only update if we got a valid JSON dictionary or list if not isinstance(response_json, dict) and not isinstance(response_json, list): raise ValueError("Response must be a JSON dictionary or list, ignoring response.") run.update_fields(response_json) message = "Webhook called successfully." except ValueError: message = "Response must be a JSON dictionary, ignoring response." if 200 <= status_code < 300: webhook_event.status = cls.STATUS_COMPLETE else: webhook_event.status = cls.STATUS_FAILED message = "Got non 200 response (%d) from webhook." % response.status_code raise Exception("Got non 200 response (%d) from webhook." % response.status_code) except Exception as e: import traceback traceback.print_exc() webhook_event.status = cls.STATUS_FAILED message = "Error calling webhook: %s" % six.text_type(e) finally: webhook_event.save() # make sure our message isn't too long if message: message = message[:255] request_time = (time.time() - start) * 1000 contact = None if webhook_event.run: contact = webhook_event.run.contact result = WebHookResult.objects.create(event=webhook_event, contact=contact, url=webhook_url, status_code=status_code, body=body, message=message, data=urlencode(data, doseq=True), request_time=request_time, created_by=api_user, modified_by=api_user) # if this is a test contact, add an entry to our action log if run.contact.is_test: log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % (reverse('api.log_read', args=[webhook_event.pk]), status_code) ActionLog.create(run, log_txt, safe=True) return result
def create_contacts(self, orgs, locations, num_total): batch_size = 5000 num_test_contacts = len(orgs) * len(USERS) group_membership_model = ContactGroup.contacts.through group_counts = defaultdict(int) self._log("Creating %d test contacts...\n" % num_test_contacts) for org in orgs: for user in org.cache['users']: Contact.get_test_contact(user) self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts)) base_contact_id = self.get_current_id(Contact) + 1 # Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed # count row for every contact with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model): names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]] names = [n if n else None for n in names] batch = 1 for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size): contacts = [] urns = [] values = [] memberships = [] def add_to_group(g): group_counts[g] += 1 memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g)) for c_index in index_batch: # pragma: no cover org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org name = self.random_choice(names) location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total) c = { 'id': base_contact_id + c_index, # database id this contact will have when created 'org': org, 'user': org.cache['users'][0], 'name': name, 'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None, 'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None, 'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'ward': location[0] if location else None, 'district': location[1] if location else None, 'state': location[2] if location else None, 'language': self.random_choice(CONTACT_LANGS), 'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB), 'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB), 'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB), 'created_on': created_on, 'modified_on': self.random_date(created_on, self.db_ends_on), } if c['is_active']: if not c['is_blocked'] and not c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL]) if c['is_blocked']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED]) if c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED]) contacts.append(Contact(org=org, name=c['name'], language=c['language'], is_stopped=c['is_stopped'], is_blocked=c['is_blocked'], is_active=c['is_active'], created_by=user, created_on=c['created_on'], modified_by=user, modified_on=c['modified_on'])) if c['tel']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME, path=c['tel'], urn=URN.from_tel(c['tel']))) if c['twitter']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME, path=c['twitter'], urn=URN.from_twitter(c['twitter']))) if c['gender']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'], string_value=c['gender'])) if c['age']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'], string_value=str(c['age']), decimal_value=c['age'])) if c['joined']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'], string_value=datetime_to_str(c['joined']), datetime_value=c['joined'])) if location: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'], string_value=c['ward'].name, location_value=c['ward'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'], string_value=c['district'].name, location_value=c['district'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'], string_value=c['state'].name, location_value=c['state'])) # let each group decide if it is taking this contact for g in org.cache['groups']: if g.member(c) if callable(g.member) else self.probability(g.member): add_to_group(g) Contact.objects.bulk_create(contacts) ContactURN.objects.bulk_create(urns) Value.objects.bulk_create(values) group_membership_model.objects.bulk_create(memberships) self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1))) batch += 1 # create group count records manually counts = [] for group, count in group_counts.items(): counts.append(ContactGroupCount(group=group, count=count, is_squashed=True)) ContactGroupCount.objects.bulk_create(counts) # for sanity check that our presumed last contact id matches the last actual contact id assert c['id'] == Contact.objects.order_by('-id').first().id