def test_twilio_failed_auth(self):
def create(self, to=None, from_=None, url=None, status_callback=None):
from twilio import TwilioRestException
raise TwilioRestException(403, 'http://twilio.com', code=20003)
MockTwilioClient.MockCalls.create = create
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN", self.admin)
self.org.save()
# import an ivr flow
self.import_file('call_me_maybe')
flow = Flow.objects.filter(name='Call me maybe').first()
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
log = ActionLog.objects.all().order_by('-pk').first()
self.assertEquals(
log.text,
'Call ended. Could not authenticate with your Twilio account. '
'Check your token and try again.')
def test_twilio_failed_auth(self):
def create(self, to=None, from_=None, url=None, status_callback=None):
from twilio import TwilioRestException
raise TwilioRestException(403, 'http://twilio.com', code=20003)
MockTwilioClient.MockCalls.create = create
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN")
self.org.save()
# import an ivr flow
self.import_file('call-me-maybe')
flow = Flow.objects.filter(name='Call me maybe').first()
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
log = ActionLog.objects.all().order_by('-pk').first()
self.assertEquals(log.text, 'Call ended. Could not authenticate with your Twilio account. '
'Check your token and try again.')
def setUp(self):
super(APITest, self).setUp()
self.joe = self.create_contact("Joe Blow", "0788123123")
self.frank = self.create_contact("Frank", twitter="franky")
self.test_contact = Contact.get_test_contact(self.user)
self.twitter = Channel.create(self.org,
self.user,
None,
'TT',
name="Twitter Channel",
address="billy_bob",
role="SR",
scheme='twitter')
self.create_secondary_org()
self.hans = self.create_contact("Hans Gruber",
"+4921551511",
org=self.org2)
self.maxDiff = None
# this is needed to prevent REST framework from rolling back transaction created around each unit test
connection.settings_dict['ATOMIC_REQUESTS'] = False
def setUp(self):
super(APITest, self).setUp()
self.joe = self.create_contact("Joe Blow", "0788123123")
self.frank = self.create_contact("Frank", twitter="franky")
self.test_contact = Contact.get_test_contact(self.user)
self.twitter = Channel.create(self.org, self.user, None, 'TT', name="Twitter Channel",
address="billy_bob", role="SR", scheme='twitter')
self.create_secondary_org()
self.hans = self.create_contact("Hans Gruber", "+4921551511", org=self.org2)
self.maxDiff = None
# this is needed to prevent REST framework from rolling back transaction created around each unit test
connection.settings_dict['ATOMIC_REQUESTS'] = False
def test_rule_first_ivr_flow(self):
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN", self.admin)
self.org.save()
# import an ivr flow
flow = self.get_flow('rule_first_ivr')
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
# start our flow
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
# should be using the usersettings number in test mode
self.assertEquals('Placing test call to +1 800-555-1212',
ActionLog.objects.all().first().text)
# we should have an outbound ivr call now
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertEquals(0, call.get_duration())
self.assertIsNotNone(call)
self.assertEquals('CallSid', call.external_id)
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid',
CallStatus='in-progress',
CallDuration=20)
response = self.client.post(
reverse('ivr.ivrcall_handle', args=[call.pk]), post_data)
self.assertContains(response, '<Say>Thanks for calling!</Say>')
# make sure a message from the person on the call goes to the
# inbox since our flow doesn't handle text messages
msg = self.create_msg(direction='I',
contact=test_contact,
text="message during phone call")
self.assertFalse(Flow.find_and_handle(msg))
def test_rule_first_ivr_flow(self):
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN")
self.org.save()
# import an ivr flow
flow = self.get_flow('rule-first-ivr')
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
# start our flow
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
# should be using the usersettings number in test mode
self.assertEquals('Placing test call to +1 800-555-1212', ActionLog.objects.all().first().text)
# we should have an outbound ivr call now
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertEquals(0, call.get_duration())
self.assertIsNotNone(call)
self.assertEquals('CallSid', call.external_id)
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid', CallStatus='in-progress', CallDuration=20)
response = self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), post_data)
self.assertContains(response, '<Say>Thanks for calling!</Say>')
# make sure a message from the person on the call goes to the
# inbox since our flow doesn't handle text messages
msg = self.create_msg(direction='I', contact=test_contact, text="message during phone call")
self.assertFalse(Flow.find_and_handle(msg))
def create_contacts(self, orgs, locations, num_contacts):
"""
Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn
id to avoid trying to hold all contact and URN objects in memory.
"""
group_counts = defaultdict(int)
self._log("Creating %d test contacts..." % (len(orgs) * len(USERS)))
for org in orgs:
test_contacts = []
for user in org.cache['users']:
test_contacts.append(Contact.get_test_contact(user))
org.cache['test_contacts'] = test_contacts
self._log(self.style.SUCCESS("OK") + '\n')
self._log("Creating %d regular contacts...\n" % num_contacts)
# disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, ContactGroup.contacts.through):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch_num = 1
for index_batch in chunk_list(six.moves.xrange(num_contacts), self.batch_size):
batch = []
# generate flat representations and contact objects for this batch
for c_index in index_batch: # pragma: no cover
org = self.random_org(orgs)
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(c_index / num_contacts)
c = {
'org': org,
'user': org.cache['users'][0],
'name': name,
'groups': [],
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
# work out which system groups this contact belongs to
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
# let each user group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
c['groups'].append(g)
# track changes to group counts
for g in c['groups']:
group_counts[g] += 1
batch.append(c)
self._create_contact_batch(batch)
self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1)))
batch_num += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
group.count = count
ContactGroupCount.objects.bulk_create(counts)
def test_ivr_flow(self):
# should be able to create an ivr flow
self.assertTrue(self.org.supports_ivr())
self.assertTrue(self.admin.groups.filter(name="Beta"))
self.assertContains(self.client.get(reverse('flows.flow_create')),
'Phone Call')
# no twilio config yet
self.assertFalse(self.org.is_connected_to_twilio())
self.assertIsNone(self.org.get_twilio_client())
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN", self.admin)
self.org.save()
self.assertTrue(self.org.is_connected_to_twilio())
self.assertIsNotNone(self.org.get_twilio_client())
# import an ivr flow
self.import_file('call_me_maybe')
# make sure our flow is there as expected
flow = Flow.objects.filter(name='Call me maybe').first()
self.assertEquals(
'callme',
flow.triggers.filter(trigger_type='K').first().keyword)
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
# start our flow as a test contact
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
call = IVRCall.objects.filter(direction=OUTGOING).first()
# should be using the usersettings number in test mode
self.assertEquals('Placing test call to +1 800-555-1212',
ActionLog.objects.all().first().text)
# explicitly hanging up on a test call should remove it
call.update_status('in-progress', 0)
call.save()
IVRCall.hangup_test_call(flow)
self.assertTrue(IVRCall.objects.filter(pk=call.pk).first())
ActionLog.objects.all().delete()
IVRCall.objects.all().delete()
# now pretend we are a normal caller
eric = self.create_contact('Eric Newcomer', number='+13603621737')
Contact.set_simulation(False)
flow.start([], [eric], restart_participants=True)
# we should have an outbound ivr call now
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertEquals(0, call.get_duration())
self.assertIsNotNone(call)
self.assertEquals('CallSid', call.external_id)
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid',
CallStatus='in-progress',
CallDuration=20)
response = self.client.post(
reverse('ivr.ivrcall_handle', args=[call.pk]), post_data)
self.assertContains(
response,
'<Say>Would you like me to call you? Press one for yes, two for no, or three for maybe.</Say>'
)
self.assertEquals(1, Msg.all_messages.filter(msg_type=IVR).count())
self.assertEquals(1, self.org.get_credits_used())
# make sure a message from the person on the call goes to the
# inbox since our flow doesn't handle text messages
msg = self.create_msg(direction='I',
contact=eric,
text="message during phone call")
self.assertFalse(Flow.find_and_handle(msg))
# updated our status and duration accordingly
call = IVRCall.objects.get(pk=call.pk)
self.assertEquals(20, call.duration)
self.assertEquals(IN_PROGRESS, call.status)
# don't press any numbers, but # instead
response = self.client.post(
reverse('ivr.ivrcall_handle', args=[call.pk]) + "?empty=1", dict())
self.assertContains(response,
'<Say>Press one, two, or three. Thanks.</Say>')
self.assertEquals(4, self.org.get_credits_used())
# press the number 4 (unexpected)
response = self.client.post(
reverse('ivr.ivrcall_handle', args=[call.pk]), dict(Digits=4))
# our inbound message should be handled
msg = Msg.current_messages.filter(
text='4', msg_type=IVR).order_by('-created_on').first()
self.assertEqual('H', msg.status)
self.assertContains(response,
'<Say>Press one, two, or three. Thanks.</Say>')
self.assertEquals(6, self.org.get_credits_used())
# two more messages, one inbound and it's response
self.assertEquals(5, Msg.all_messages.filter(msg_type=IVR).count())
# now let's have them press the number 3 (for maybe)
response = self.client.post(
reverse('ivr.ivrcall_handle', args=[call.pk]), dict(Digits=3))
self.assertContains(response, '<Say>This might be crazy.</Say>')
messages = Msg.all_messages.filter(msg_type=IVR).order_by('pk')
self.assertEquals(7, messages.count())
self.assertEquals(8, self.org.get_credits_used())
for msg in messages:
self.assertEquals(1,
msg.steps.all().count(),
msg="Message '%s' not attached to step" %
msg.text)
# twilio would then disconnect the user and notify us of a completed call
self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]),
dict(CallStatus='completed'))
call = IVRCall.objects.get(pk=call.pk)
self.assertEquals(COMPLETED, call.status)
self.assertFalse(FlowRun.objects.filter(call=call).first().is_active)
# simulation gets flipped off by middleware, and this unhandled message doesn't flip it back on
self.assertFalse(Contact.get_simulation())
# also shouldn't have any ActionLogs for non-test users
self.assertEquals(0, ActionLog.objects.all().count())
self.assertEquals(1, flow.get_completed_runs())
# should still have no active runs
self.assertEquals(0, FlowRun.objects.filter(is_active=True).count())
# and we've exited the flow
step = FlowStep.objects.all().order_by('-pk').first()
self.assertTrue(step.left_on)
# test other our call status mappings with twilio
def test_status_update(call_to_update, twilio_status, temba_status):
call_to_update.update_status(twilio_status, 0)
call_to_update.save()
self.assertEquals(temba_status,
IVRCall.objects.get(pk=call_to_update.pk).status)
test_status_update(call, 'queued', QUEUED)
test_status_update(call, 'ringing', RINGING)
test_status_update(call, 'canceled', CANCELED)
test_status_update(call, 'busy', BUSY)
test_status_update(call, 'failed', FAILED)
test_status_update(call, 'no-answer', NO_ANSWER)
FlowStep.objects.all().delete()
IVRCall.objects.all().delete()
# try sending callme trigger
from temba.msgs.models import INCOMING
msg = self.create_msg(direction=INCOMING, contact=eric, text="callme")
# make sure if we are started with a message we still create a normal voice run
flow.start([], [eric], restart_participants=True, start_msg=msg)
# we should have an outbound ivr call now, and no steps yet
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertIsNotNone(call)
self.assertEquals(0, FlowStep.objects.all().count())
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid',
CallStatus='in-progress',
CallDuration=20)
self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]),
post_data)
# should have two flow steps (the outgoing messages, and the step to handle the response)
steps = FlowStep.objects.all().order_by('pk')
# the first step has exactly one message which is an outgoing IVR message
self.assertEquals(1, steps.first().messages.all().count())
self.assertEquals(
1,
steps.first().messages.filter(direction=OUTGOING,
msg_type=IVR).count())
# the next step shouldn't have any messages yet since they haven't pressed anything
self.assertEquals(0, steps[1].messages.all().count())
# try updating our status to completed for a test contact
Contact.set_simulation(True)
flow.start([], [test_contact])
call = IVRCall.objects.filter(
direction=OUTGOING).order_by('-pk').first()
call.update_status('completed', 30)
call.save()
call.refresh_from_db()
self.assertEqual(ActionLog.objects.all().order_by('-pk').first().text,
'Call ended.')
self.assertEqual(call.duration, 30)
# now look at implied duration
call.update_status('in-progress', None)
call.save()
call.refresh_from_db()
self.assertIsNotNone(call.get_duration())
def test_ivr_flow(self):
# should be able to create an ivr flow
self.assertTrue(self.org.supports_ivr())
self.assertTrue(self.admin.groups.filter(name="Beta"))
self.assertContains(self.client.get(reverse('flows.flow_create')), 'Phone Call')
# no twilio config yet
self.assertFalse(self.org.is_connected_to_twilio())
self.assertIsNone(self.org.get_twilio_client())
# connect it and check our client is configured
self.org.connect_twilio("TEST_SID", "TEST_TOKEN")
self.org.save()
self.assertTrue(self.org.is_connected_to_twilio())
self.assertIsNotNone(self.org.get_twilio_client())
# import an ivr flow
self.import_file('call-me-maybe')
# make sure our flow is there as expected
flow = Flow.objects.filter(name='Call me maybe').first()
self.assertEquals('callme', flow.triggers.filter(trigger_type='K').first().keyword)
user_settings = self.admin.get_settings()
user_settings.tel = '+18005551212'
user_settings.save()
# start our flow as a test contact
test_contact = Contact.get_test_contact(self.admin)
Contact.set_simulation(True)
flow.start([], [test_contact])
call = IVRCall.objects.filter(direction=OUTGOING).first()
# should be using the usersettings number in test mode
self.assertEquals('Placing test call to +1 800-555-1212', ActionLog.objects.all().first().text)
# explicitly hanging up on a test call should remove it
call.update_status('in-progress', 0)
call.save()
IVRCall.hangup_test_call(flow)
self.assertIsNone(IVRCall.objects.filter(pk=call.pk).first())
ActionLog.objects.all().delete()
IVRCall.objects.all().delete()
# now pretend we are a normal caller
eric = self.create_contact('Eric Newcomer', number='+13603621737')
Contact.set_simulation(False)
flow.start([], [eric], restart_participants=True)
# we should have an outbound ivr call now
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertEquals(0, call.get_duration())
self.assertIsNotNone(call)
self.assertEquals('CallSid', call.external_id)
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid', CallStatus='in-progress', CallDuration=20)
response = self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), post_data)
self.assertContains(response, '<Say>Would you like me to call you? Press one for yes, two for no, or three for maybe.</Say>')
self.assertEquals(1, Msg.all_messages.filter(msg_type=IVR).count())
self.assertEquals(1, self.org.get_credits_used())
# make sure a message from the person on the call goes to the
# inbox since our flow doesn't handle text messages
msg = self.create_msg(direction='I', contact=eric, text="message during phone call")
self.assertFalse(Flow.find_and_handle(msg))
# updated our status and duration accordingly
call = IVRCall.objects.get(pk=call.pk)
self.assertEquals(20, call.duration)
self.assertEquals(IN_PROGRESS, call.status)
# press the number 4 (unexpected)
response = self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), dict(Digits=4))
self.assertContains(response, '<Say>Press one, two, or three. Thanks.</Say>')
self.assertEquals(4, self.org.get_credits_used())
# two more messages, one inbound and it's response
self.assertEquals(3, Msg.all_messages.filter(msg_type=IVR).count())
# now let's have them press the number 3 (for maybe)
response = self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), dict(Digits=3))
self.assertContains(response, '<Say>This might be crazy.</Say>')
messages = Msg.all_messages.filter(msg_type=IVR).order_by('pk')
self.assertEquals(5, messages.count())
self.assertEquals(6, self.org.get_credits_used())
for msg in messages:
self.assertEquals(1, msg.steps.all().count(), msg="Message '%s' not attached to step" % msg.text)
# twilio would then disconnect the user and notify us of a completed call
self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), dict(CallStatus='completed'))
call = IVRCall.objects.get(pk=call.pk)
self.assertEquals(COMPLETED, call.status)
self.assertFalse(FlowRun.objects.filter(call=call).first().is_active)
# simulation gets flipped off by middleware, and this unhandled message doesn't flip it back on
self.assertFalse(Contact.get_simulation())
# also shouldn't have any ActionLogs for non-test users
self.assertEquals(0, ActionLog.objects.all().count())
self.assertEquals(1, flow.get_completed_runs())
# should still have no active runs
self.assertEquals(0, FlowRun.objects.filter(is_active=True).count())
# and we've exited the flow
step = FlowStep.objects.all().order_by('-pk').first()
self.assertTrue(step.left_on)
# test other our call status mappings with twilio
def test_status_update(call_to_update, twilio_status, temba_status):
call_to_update.update_status(twilio_status, 0)
call_to_update.save()
self.assertEquals(temba_status, IVRCall.objects.get(pk=call_to_update.pk).status)
test_status_update(call, 'queued', QUEUED)
test_status_update(call, 'ringing', RINGING)
test_status_update(call, 'canceled', CANCELED)
test_status_update(call, 'busy', BUSY)
test_status_update(call, 'failed', FAILED)
test_status_update(call, 'no-answer', NO_ANSWER)
FlowStep.objects.all().delete()
IVRCall.objects.all().delete()
# try sending callme trigger
from temba.msgs.models import INCOMING
msg = self.create_msg(direction=INCOMING, contact=eric, text="callme")
# make sure if we are started with a message we still create a normal voice run
flow.start([], [eric], restart_participants=True, start_msg=msg)
# we should have an outbound ivr call now, and no steps yet
call = IVRCall.objects.filter(direction=OUTGOING).first()
self.assertIsNotNone(call)
self.assertEquals(0, FlowStep.objects.all().count())
# after a call is picked up, twilio will call back to our server
post_data = dict(CallSid='CallSid', CallStatus='in-progress', CallDuration=20)
self.client.post(reverse('ivr.ivrcall_handle', args=[call.pk]), post_data)
# should have two flow steps (the outgoing messages, and the step to handle the response)
steps = FlowStep.objects.all().order_by('pk')
# the first step has exactly one message which is an outgoing IVR message
self.assertEquals(1, steps.first().messages.all().count())
self.assertEquals(1, steps.first().messages.filter(direction=OUTGOING, msg_type=IVR).count())
# the next step shouldn't have any messages yet since they haven't pressed anything
self.assertEquals(0, steps[1].messages.all().count())
# try updating our status to completed for a test contact
Contact.set_simulation(True)
flow.start([], [test_contact])
call = IVRCall.objects.filter(direction=OUTGOING).order_by('-pk').first()
call.update_status('completed', 30)
call.save()
call.refresh_from_db()
self.assertEqual(ActionLog.objects.all().order_by('-pk').first().text, 'Call ended.')
self.assertEqual(call.duration, 30)
# now look at implied duration
call.update_status('in-progress', None)
call.save()
call.refresh_from_db()
self.assertIsNotNone(call.get_duration())
def create_contacts(self, orgs, locations, num_contacts):
"""
Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn
id to avoid trying to hold all contact and URN objects in memory.
"""
group_counts = defaultdict(int)
self._log("Creating %d test contacts..." % (len(orgs) * len(USERS)))
for org in orgs:
test_contacts = []
for user in org.cache["users"]:
test_contacts.append(Contact.get_test_contact(user))
org.cache["test_contacts"] = test_contacts
self._log(self.style.SUCCESS("OK") + "\n")
self._log("Creating %d regular contacts...\n" % num_contacts)
# disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, ContactGroup.contacts.through):
names = [("%s %s" % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch_num = 1
for index_batch in chunk_list(range(num_contacts), self.batch_size):
batch = []
# generate flat representations and contact objects for this batch
for c_index in index_batch: # pragma: no cover
org = self.random_org(orgs)
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(c_index / num_contacts)
c = {
"org": org,
"user": org.cache["users"][0],
"name": name,
"groups": [],
"tel": "+2507%08d" % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
"twitter": "%s%d" % (name.replace(" ", "_").lower() if name else "tweep", c_index)
if self.probability(CONTACT_HAS_TWITTER_PROB)
else None,
"gender": self.random_choice(("M", "F")) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"age": self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"joined": self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"ward": location[0] if location else None,
"district": location[1] if location else None,
"state": location[2] if location else None,
"language": self.random_choice(CONTACT_LANGS),
"is_stopped": self.probability(CONTACT_IS_STOPPED_PROB),
"is_blocked": self.probability(CONTACT_IS_BLOCKED_PROB),
"is_active": self.probability(1 - CONTACT_IS_DELETED_PROB),
"created_on": created_on,
"modified_on": self.random_date(created_on, self.db_ends_on),
}
c["fields_as_json"] = {}
if c["gender"] is not None:
c["fields_as_json"][str(org.cache["fields"]["gender"].uuid)] = {"text": str(c["gender"])}
if c["age"] is not None:
c["fields_as_json"][str(org.cache["fields"]["age"].uuid)] = {
"text": str(c["age"]),
"number": str(c["age"]),
}
if c["joined"] is not None:
c["fields_as_json"][str(org.cache["fields"]["joined"].uuid)] = {
"text": org.format_datetime(c["joined"], show_time=False),
"datetime": timezone.localtime(c["joined"], org.timezone).isoformat(),
}
if location:
c["fields_as_json"].update(
{
str(org.cache["fields"]["ward"].uuid): {
"text": str(c["ward"].path.split(" > ")[-1]),
"ward": c["ward"].path,
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["district"].uuid): {
"text": str(c["district"].path.split(" > ")[-1]),
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["state"].uuid): {
"text": str(c["state"].path.split(" > ")[-1]),
"state": c["state"].path,
},
}
)
# work out which system groups this contact belongs to
if c["is_active"]:
if not c["is_blocked"] and not c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_ALL])
if c["is_blocked"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_BLOCKED])
if c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_STOPPED])
# let each user group decide if it is taking this contact
for g in org.cache["groups"]:
if g.member(c) if callable(g.member) else self.probability(g.member):
c["groups"].append(g)
# track changes to group counts
for g in c["groups"]:
group_counts[g] += 1
batch.append(c)
self._create_contact_batch(batch)
self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1)))
batch_num += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
group.count = count
ContactGroupCount.objects.bulk_create(counts)
def create_contacts(self, orgs, locations, num_contacts):
"""
Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn
id to avoid trying to hold all contact and URN objects in memory.
"""
group_counts = defaultdict(int)
self._log("Creating %d test contacts..." % (len(orgs) * len(USERS)))
for org in orgs:
test_contacts = []
for user in org.cache["users"]:
test_contacts.append(Contact.get_test_contact(user))
org.cache["test_contacts"] = test_contacts
self._log(self.style.SUCCESS("OK") + "\n")
self._log("Creating %d regular contacts...\n" % num_contacts)
# disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, ContactGroup.contacts.through):
names = [("%s %s" % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch_num = 1
for index_batch in chunk_list(range(num_contacts), self.batch_size):
batch = []
# generate flat representations and contact objects for this batch
for c_index in index_batch: # pragma: no cover
org = self.random_org(orgs)
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(c_index / num_contacts)
c = {
"org": org,
"user": org.cache["users"][0],
"name": name,
"groups": [],
"tel": "+2507%08d" % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
"twitter": "%s%d" % (name.replace(" ", "_").lower() if name else "tweep", c_index)
if self.probability(CONTACT_HAS_TWITTER_PROB)
else None,
"gender": self.random_choice(("M", "F")) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"age": self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"joined": self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"ward": location[0] if location else None,
"district": location[1] if location else None,
"state": location[2] if location else None,
"language": self.random_choice(CONTACT_LANGS),
"is_stopped": self.probability(CONTACT_IS_STOPPED_PROB),
"is_blocked": self.probability(CONTACT_IS_BLOCKED_PROB),
"is_active": self.probability(1 - CONTACT_IS_DELETED_PROB),
"created_on": created_on,
"modified_on": self.random_date(created_on, self.db_ends_on),
}
c["fields_as_json"] = {}
if c["gender"] is not None:
c["fields_as_json"][str(org.cache["fields"]["gender"].uuid)] = {"text": str(c["gender"])}
if c["age"] is not None:
c["fields_as_json"][str(org.cache["fields"]["age"].uuid)] = {
"text": str(c["age"]),
"number": str(c["age"]),
}
if c["joined"] is not None:
c["fields_as_json"][str(org.cache["fields"]["joined"].uuid)] = {
"text": org.format_datetime(c["joined"], show_time=False),
"datetime": timezone.localtime(c["joined"], org.timezone).isoformat(),
}
if location:
c["fields_as_json"].update(
{
str(org.cache["fields"]["ward"].uuid): {
"text": str(c["ward"].path.split(" > ")[-1]),
"ward": c["ward"].path,
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["district"].uuid): {
"text": str(c["district"].path.split(" > ")[-1]),
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["state"].uuid): {
"text": str(c["state"].path.split(" > ")[-1]),
"state": c["state"].path,
},
}
)
# work out which system groups this contact belongs to
if c["is_active"]:
if not c["is_blocked"] and not c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_ALL])
if c["is_blocked"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_BLOCKED])
if c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_STOPPED])
# let each user group decide if it is taking this contact
for g in org.cache["groups"]:
if g.member(c) if callable(g.member) else self.probability(g.member):
c["groups"].append(g)
# track changes to group counts
for g in c["groups"]:
group_counts[g] += 1
batch.append(c)
self._create_contact_batch(batch)
self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1)))
batch_num += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
group.count = count
ContactGroupCount.objects.bulk_create(counts)
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
