def msg_as_task(msg):
"""
Used to serialize msgs as tasks to courier
"""
msg_json = dict(
id=msg.id,
uuid=str(msg.uuid) if msg.uuid else "",
org_id=msg.org_id,
channel_id=msg.channel_id,
channel_uuid=msg.channel.uuid,
contact_id=msg.contact_id,
contact_urn_id=msg.contact_urn_id,
status=msg.status,
direction=msg.direction,
text=msg.text,
high_priority=msg.high_priority,
priority=500 if msg.high_priority else
100, # TODO stop using this on courier side and then remove
urn=msg.contact_urn.urn,
error_count=msg.error_count,
attachments=msg.attachments,
response_to_id=msg.response_to_id,
external_id=msg.external_id,
tps_cost=msg.channel.calculate_tps_cost(msg),
next_attempt=datetime_to_str(msg.next_attempt, ms=True),
created_on=datetime_to_str(msg.created_on, ms=True),
modified_on=datetime_to_str(msg.modified_on, ms=True),
queued_on=datetime_to_str(msg.queued_on, ms=True),
sent_on=datetime_to_str(msg.sent_on, ms=True))
if msg.contact_urn.auth: # pragma: no cover
msg_json['contact_urn_auth'] = msg.contact_urn.auth
return msg_json
def msg_as_task(msg):
"""
Used to serialize msgs as tasks to courier
"""
msg_json = dict(id=msg.id,
uuid=unicode(msg.uuid) if msg.uuid else "",
org_id=msg.org_id,
channel_id=msg.channel_id,
channel_uuid=msg.channel.uuid,
contact_id=msg.contact_id,
contact_urn_id=msg.contact_urn_id,
status=msg.status,
direction=msg.direction,
text=msg.text,
priority=msg.priority,
urn=msg.contact_urn.urn,
error_count=msg.error_count,
attachments=msg.attachments,
response_to_id=msg.response_to_id,
external_id=msg.external_id,
next_attempt=datetime_to_str(msg.next_attempt, ms=True),
created_on=datetime_to_str(msg.created_on, ms=True),
modified_on=datetime_to_str(msg.modified_on, ms=True),
queued_on=datetime_to_str(msg.queued_on, ms=True),
sent_on=datetime_to_str(msg.sent_on, ms=True))
if msg.contact_urn.auth: # pragma: no cover
msg_json['contact_urn_auth'] = msg.contact_urn.auth
return msg_json
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'], created_on=c['created_on'],
modified_by=c['user'], modified_on=c['modified_on'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_values = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if c['ward']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
if c['district']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
if c['state']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
for g in c['groups']:
batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
Value.objects.bulk_create(batch_values)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
# allow this much absolute change from previous results (milliseconds)
ALLOWED_CHANGE_MAXIMUM = 50
# allow this much percentage change from previous results
ALLOWED_CHANGE_PERCENTAGE = 5
# a org specific context used in URL generation
URL_CONTEXT_TEMPLATE = {
'first-group':
lambda org: ContactGroup.user_groups.filter(org=org).order_by('id').first(
).uuid,
'last-group':
lambda org: ContactGroup.user_groups.filter(org=org).order_by('-id').first(
).uuid,
'1-year-ago':
lambda org: datetime_to_str(now() - timedelta(days=365),
get_datetime_format(org.get_dayfirst())[0])
}
TEST_URLS = (
'/api/v2/channels.json',
'/api/v2/channel_events.json',
'/api/v2/contacts.json',
'/api/v2/contacts.json?deleted=true',
'/api/v2/contacts.json?group={first-group}',
'/api/v2/groups.json',
'/api/v2/fields.json',
'/api/v2/labels.json',
'/api/v2/messages.json?folder=incoming',
'/api/v2/messages.json?folder=inbox',
'/api/v2/messages.json?folder=flows',
'/api/v2/messages.json?folder=archived',
def trigger_flow_event(cls,
webhook_url,
flow,
run,
node_uuid,
contact,
event,
action='POST',
resthook=None):
org = flow.org
api_user = get_api_user()
json_time = datetime_to_str(timezone.now())
# get the results for this contact
results = flow.get_results(contact)
values = []
if results and results[0]:
values = results[0]['values']
for value in values:
value['time'] = datetime_to_str(value['time'])
value['value'] = unicode(value['value'])
# if the action is on the first node
# we might not have an sms (or channel) yet
channel = None
text = None
contact_urn = contact.get_urn()
if event:
text = event.text
channel = event.channel
contact_urn = event.contact_urn
if channel:
channel_id = channel.pk
else:
channel_id = -1
steps = []
for step in run.steps.prefetch_related(
'messages', 'broadcasts').order_by('arrived_on'):
steps.append(
dict(type=step.step_type,
node=step.step_uuid,
arrived_on=datetime_to_str(step.arrived_on),
left_on=datetime_to_str(step.left_on),
text=step.get_text(),
value=step.rule_value))
data = dict(channel=channel_id,
relayer=channel_id,
flow=flow.id,
flow_name=flow.name,
flow_base_language=flow.base_language,
run=run.id,
text=text,
step=unicode(node_uuid),
phone=contact.get_urn_display(org=org,
scheme=TEL_SCHEME,
formatted=False),
contact=contact.uuid,
urn=unicode(contact_urn),
values=json.dumps(values),
steps=json.dumps(steps),
time=json_time)
if not action:
action = 'POST'
webhook_event = WebHookEvent.objects.create(org=org,
event=FLOW,
channel=channel,
data=json.dumps(data),
try_count=1,
action=action,
resthook=resthook,
created_by=api_user,
modified_by=api_user)
status_code = -1
message = "None"
body = None
# webhook events fire immediately since we need the results back
try:
# only send webhooks when we are configured to, otherwise fail
if not settings.SEND_WEBHOOKS:
raise Exception(
"!! Skipping WebHook send, SEND_WEBHOOKS set to False")
# no url, bail!
if not webhook_url:
raise Exception("No webhook_url specified, skipping send")
# some hosts deny generic user agents, use Temba as our user agent
if action == 'GET':
response = requests.get(webhook_url,
headers=TEMBA_HEADERS,
timeout=10)
else:
response = requests.post(webhook_url,
data=data,
headers=TEMBA_HEADERS,
timeout=10)
response_text = response.text
body = response.text
status_code = response.status_code
if response.status_code == 200 or response.status_code == 201:
try:
response_json = json.loads(response_text)
# only update if we got a valid JSON dictionary or list
if not isinstance(response_json, dict) and not isinstance(
response_json, list):
raise ValueError(
"Response must be a JSON dictionary or list, ignoring response."
)
run.update_fields(response_json)
message = "Webhook called successfully."
except ValueError as e:
message = "Response must be a JSON dictionary, ignoring response."
webhook_event.status = COMPLETE
else:
webhook_event.status = FAILED
message = "Got non 200 response (%d) from webhook." % response.status_code
raise Exception("Got non 200 response (%d) from webhook." %
response.status_code)
except Exception as e:
import traceback
traceback.print_exc()
webhook_event.status = FAILED
message = "Error calling webhook: %s" % unicode(e)
finally:
webhook_event.save()
# make sure our message isn't too long
if message:
message = message[:255]
result = WebHookResult.objects.create(event=webhook_event,
url=webhook_url,
status_code=status_code,
body=body,
message=message,
data=urlencode(data,
doseq=True),
created_by=api_user,
modified_by=api_user)
# if this is a test contact, add an entry to our action log
if run.contact.is_test:
from temba.flows.models import ActionLog
log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % (
reverse('api.log_read', args=[webhook_event.pk
]), status_code)
ActionLog.create(run, log_txt, safe=True)
return result
def trigger_flow_event(cls, webhook_url, flow, run, node, contact, event, action="POST"):
org = flow.org
api_user = get_api_user()
# no-op if no webhook configured
if not webhook_url:
return
json_time = datetime_to_str(timezone.now())
# get the results for this contact
results = flow.get_results(contact)
values = []
if results and results[0]:
values = results[0]["values"]
for value in values:
value["time"] = datetime_to_str(value["time"])
value["value"] = unicode(value["value"])
# if the action is on the first node
# we might not have an sms (or channel) yet
channel = None
text = None
if event:
text = event.text
channel = event.channel
if channel:
channel_id = channel.pk
else:
channel_id = -1
steps = []
for step in run.steps.all().order_by("arrived_on"):
steps.append(
dict(
type=step.step_type,
node=step.step_uuid,
arrived_on=datetime_to_str(step.arrived_on),
left_on=datetime_to_str(step.left_on),
text=step.get_text(),
value=step.rule_value,
)
)
data = dict(
channel=channel_id,
relayer=channel_id,
flow=flow.id,
run=run.id,
text=text,
step=unicode(node.uuid),
phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, full=True),
values=json.dumps(values),
steps=json.dumps(steps),
time=json_time,
)
if not action:
action = "POST"
webhook_event = WebHookEvent.objects.create(
org=org,
event=FLOW,
channel=channel,
data=json.dumps(data),
try_count=1,
action=action,
created_by=api_user,
modified_by=api_user,
)
status_code = -1
message = "None"
body = None
# webhook events fire immediately since we need the results back
try:
# only send webhooks when we are configured to, otherwise fail
if not settings.SEND_WEBHOOKS:
raise Exception("!! Skipping WebHook send, SEND_WEBHOOKS set to False")
# some hosts deny generic user agents, use Temba as our user agent
if action == "GET":
response = requests.get(webhook_url, headers=TEMBA_HEADERS, timeout=10)
else:
response = requests.post(webhook_url, data=data, headers=TEMBA_HEADERS, timeout=10)
response_text = response.text
body = response.text
status_code = response.status_code
if response.status_code == 200 or response.status_code == 201:
try:
response_json = json.loads(response_text)
# only update if we got a valid JSON dictionary
if not isinstance(response_json, dict):
raise ValueError("Response must be a JSON dictionary, ignoring response.")
run.update_fields(response_json)
message = "Webhook called successfully."
except ValueError as e:
message = "Response must be a JSON dictionary, ignoring response."
webhook_event.status = COMPLETE
else:
webhook_event.status = FAILED
message = "Got non 200 response (%d) from webhook." % response.status_code
raise Exception("Got non 200 response (%d) from webhook." % response.status_code)
except Exception as e:
import traceback
traceback.print_exc()
webhook_event.status = FAILED
message = "Error calling webhook: %s" % unicode(e)
finally:
webhook_event.save()
result = WebHookResult.objects.create(
event=webhook_event,
url=webhook_url,
status_code=status_code,
body=body,
message=message,
data=urlencode(data, doseq=True),
created_by=api_user,
modified_by=api_user,
)
# if this is a test contact, add an entry to our action log
if run.contact.is_test:
from temba.flows.models import ActionLog
log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % (
reverse("api.log_read", args=[webhook_event.pk]),
status_code,
)
ActionLog.create(run, log_txt, safe=True)
return result
def trigger_flow_event(cls, run, webhook_url, node_uuid, msg, action='POST', resthook=None, header=None):
flow = run.flow
org = flow.org
contact = run.contact
api_user = get_api_user()
json_time = datetime_to_str(timezone.now())
# get the results for this contact
results = run.flow.get_results(run.contact)
values = []
if results and results[0]:
values = results[0]['values']
for value in values:
value['time'] = datetime_to_str(value['time'])
value['value'] = six.text_type(value['value'])
if msg:
text = msg.text
attachments = msg.get_attachments()
channel = msg.channel
contact_urn = msg.contact_urn
else:
# if the action is on the first node we might not have an sms (or channel) yet
channel = None
text = None
attachments = []
contact_urn = contact.get_urn()
steps = []
for step in run.steps.prefetch_related('messages', 'broadcasts').order_by('arrived_on'):
steps.append(dict(type=step.step_type,
node=step.step_uuid,
arrived_on=datetime_to_str(step.arrived_on),
left_on=datetime_to_str(step.left_on),
text=step.get_text(),
value=step.rule_value))
data = dict(channel=channel.id if channel else -1,
channel_uuid=channel.uuid if channel else None,
relayer=channel.id if channel else -1,
flow=flow.id,
flow_uuid=flow.uuid,
flow_name=flow.name,
flow_base_language=flow.base_language,
run=run.id,
text=text,
attachments=[a.url for a in attachments],
step=six.text_type(node_uuid),
phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, formatted=False),
contact=contact.uuid,
contact_name=contact.name,
urn=six.text_type(contact_urn),
values=json.dumps(values),
steps=json.dumps(steps),
time=json_time,
header=header)
if not action: # pragma: needs cover
action = 'POST'
webhook_event = cls.objects.create(org=org, event=cls.TYPE_FLOW, channel=channel, data=json.dumps(data),
run=run, try_count=1, action=action, resthook=resthook,
created_by=api_user, modified_by=api_user)
status_code = -1
message = "None"
body = None
start = time.time()
# webhook events fire immediately since we need the results back
try:
# no url, bail!
if not webhook_url:
raise Exception("No webhook_url specified, skipping send")
# only send webhooks when we are configured to, otherwise fail
if settings.SEND_WEBHOOKS:
requests_headers = TEMBA_HEADERS
if header:
requests_headers.update(header)
# some hosts deny generic user agents, use Temba as our user agent
if action == 'GET':
response = requests.get(webhook_url, headers=requests_headers, timeout=10)
else:
response = requests.post(webhook_url, data=data, headers=requests_headers, timeout=10)
body = response.text
if body:
body = body.strip()
status_code = response.status_code
else:
print("!! Skipping WebHook send, SEND_WEBHOOKS set to False")
body = 'Skipped actual send'
status_code = 200
# process the webhook response
try:
response_json = json.loads(body, object_pairs_hook=OrderedDict)
# only update if we got a valid JSON dictionary or list
if not isinstance(response_json, dict) and not isinstance(response_json, list):
raise ValueError("Response must be a JSON dictionary or list, ignoring response.")
run.update_fields(response_json)
message = "Webhook called successfully."
except ValueError:
message = "Response must be a JSON dictionary, ignoring response."
if 200 <= status_code < 300:
webhook_event.status = cls.STATUS_COMPLETE
else:
webhook_event.status = cls.STATUS_FAILED
message = "Got non 200 response (%d) from webhook." % response.status_code
raise Exception("Got non 200 response (%d) from webhook." % response.status_code)
except Exception as e:
import traceback
traceback.print_exc()
webhook_event.status = cls.STATUS_FAILED
message = "Error calling webhook: %s" % six.text_type(e)
finally:
webhook_event.save()
# make sure our message isn't too long
if message:
message = message[:255]
request_time = (time.time() - start) * 1000
contact = None
if webhook_event.run:
contact = webhook_event.run.contact
result = WebHookResult.objects.create(event=webhook_event,
contact=contact,
url=webhook_url,
status_code=status_code,
body=body,
message=message,
data=urlencode(data, doseq=True),
request_time=request_time,
created_by=api_user,
modified_by=api_user)
# if this is a test contact, add an entry to our action log
if run.contact.is_test:
log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % (reverse('api.log_read', args=[webhook_event.pk]), status_code)
ActionLog.create(run, log_txt, safe=True)
return result
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
