def _create_values(self, contacts, field, callback):
"""
Creates a field value for each given contact (a lot faster than calling set_field)
"""
values = []
for contact in contacts:
string_value = callback(contact)
values.append(Value(contact=contact, org=self.org, contact_field=field, string_value=string_value))
Value.objects.bulk_create(values)
return values
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'], created_on=c['created_on'],
modified_by=c['user'], modified_on=c['modified_on'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_values = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if c['ward']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
if c['district']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
if c['state']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
for g in c['groups']:
batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
Value.objects.bulk_create(batch_values)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
