def _create_values(self, contacts, field, callback): """ Creates a field value for each given contact (a lot faster than calling set_field) """ values = [] for contact in contacts: string_value = callback(contact) values.append(Value(contact=contact, org=self.org, contact_field=field, string_value=string_value)) Value.objects.bulk_create(values) return values
def _create_contact_batch(self, batch): """ Bulk creates a batch of contacts from flat representations """ for c in batch: c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'], is_stopped=c['is_stopped'], is_blocked=c['is_blocked'], is_active=c['is_active'], created_by=c['user'], created_on=c['created_on'], modified_by=c['user'], modified_on=c['modified_on']) Contact.objects.bulk_create([c['object'] for c in batch]) # now that contacts have pks, bulk create the actual URN, value and group membership objects batch_urns = [] batch_values = [] batch_memberships = [] for c in batch: org = c['org'] c['urns'] = [] if c['tel']: c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME, path=c['tel'], urn=URN.from_tel(c['tel']))) if c['twitter']: c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME, path=c['twitter'], urn=URN.from_twitter(c['twitter']))) if c['gender']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'], string_value=c['gender'])) if c['age']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'], string_value=str(c['age']), decimal_value=c['age'])) if c['joined']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'], string_value=datetime_to_str(c['joined']), datetime_value=c['joined'])) if c['ward']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'], string_value=c['ward'].name, location_value=c['ward'])) if c['district']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'], string_value=c['district'].name, location_value=c['district'])) if c['state']: batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'], string_value=c['state'].name, location_value=c['state'])) for g in c['groups']: batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g)) batch_urns += c['urns'] ContactURN.objects.bulk_create(batch_urns) Value.objects.bulk_create(batch_values) ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def create_contacts(self, orgs, locations, num_total): batch_size = 5000 num_test_contacts = len(orgs) * len(USERS) group_membership_model = ContactGroup.contacts.through group_counts = defaultdict(int) self._log("Creating %d test contacts...\n" % num_test_contacts) for org in orgs: for user in org.cache['users']: Contact.get_test_contact(user) self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts)) base_contact_id = self.get_current_id(Contact) + 1 # Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed # count row for every contact with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model): names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]] names = [n if n else None for n in names] batch = 1 for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size): contacts = [] urns = [] values = [] memberships = [] def add_to_group(g): group_counts[g] += 1 memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g)) for c_index in index_batch: # pragma: no cover org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org name = self.random_choice(names) location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total) c = { 'id': base_contact_id + c_index, # database id this contact will have when created 'org': org, 'user': org.cache['users'][0], 'name': name, 'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None, 'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None, 'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'ward': location[0] if location else None, 'district': location[1] if location else None, 'state': location[2] if location else None, 'language': self.random_choice(CONTACT_LANGS), 'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB), 'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB), 'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB), 'created_on': created_on, 'modified_on': self.random_date(created_on, self.db_ends_on), } if c['is_active']: if not c['is_blocked'] and not c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL]) if c['is_blocked']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED]) if c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED]) contacts.append(Contact(org=org, name=c['name'], language=c['language'], is_stopped=c['is_stopped'], is_blocked=c['is_blocked'], is_active=c['is_active'], created_by=user, created_on=c['created_on'], modified_by=user, modified_on=c['modified_on'])) if c['tel']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME, path=c['tel'], urn=URN.from_tel(c['tel']))) if c['twitter']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME, path=c['twitter'], urn=URN.from_twitter(c['twitter']))) if c['gender']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'], string_value=c['gender'])) if c['age']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'], string_value=str(c['age']), decimal_value=c['age'])) if c['joined']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'], string_value=datetime_to_str(c['joined']), datetime_value=c['joined'])) if location: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'], string_value=c['ward'].name, location_value=c['ward'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'], string_value=c['district'].name, location_value=c['district'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'], string_value=c['state'].name, location_value=c['state'])) # let each group decide if it is taking this contact for g in org.cache['groups']: if g.member(c) if callable(g.member) else self.probability(g.member): add_to_group(g) Contact.objects.bulk_create(contacts) ContactURN.objects.bulk_create(urns) Value.objects.bulk_create(values) group_membership_model.objects.bulk_create(memberships) self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1))) batch += 1 # create group count records manually counts = [] for group, count in group_counts.items(): counts.append(ContactGroupCount(group=group, count=count, is_squashed=True)) ContactGroupCount.objects.bulk_create(counts) # for sanity check that our presumed last contact id matches the last actual contact id assert c['id'] == Contact.objects.order_by('-id').first().id