def omnibox_results_to_dict(org, results): """ Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text} """ formatted = [] groups = [r for r in results if isinstance(r, ContactGroup)] group_counts = ContactGroupCount.get_totals(groups) if groups else {} for obj in results: if isinstance(obj, ContactGroup): result = { 'id': 'g-%s' % obj.uuid, 'text': obj.name, 'extra': group_counts[obj] } elif isinstance(obj, Contact): result = { 'id': 'c-%s' % obj.uuid, 'text': obj.get_display(org) } elif isinstance(obj, ContactURN): result = { 'id': 'u-%d' % obj.id, 'text': obj.get_display(org), 'extra': obj.contact.name or None, 'scheme': obj.scheme } formatted.append(result) return formatted
def omnibox_results_to_dict(org, results): """ Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text} """ formatted = [] groups = [r for r in results if isinstance(r, ContactGroup)] group_counts = ContactGroupCount.get_totals(groups) if groups else {} for obj in results: if isinstance(obj, ContactGroup): result = {"id": "g-%s" % obj.uuid, "text": obj.name, "extra": group_counts[obj]} elif isinstance(obj, Contact): if org.is_anon: result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org)} else: result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org), "extra": obj.get_urn_display()} elif isinstance(obj, ContactURN): result = { "id": "u-%d" % obj.id, "text": obj.get_display(org), "extra": obj.contact.name or None, "scheme": obj.scheme, } formatted.append(result) return formatted
def populate_contactgroupcounts(apps, schema_editor): from temba.contacts.models import ContactGroup, ContactGroupCount for group_id in ContactGroup.all_groups.all().values_list('id', flat=True): group = ContactGroup.all_groups.get(id=group_id) count = ContactGroupCount.populate_for_group(group) print "%s %d" % (group.name, count.count)
def omnibox_results_to_dict(org, results): """ Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text} """ formatted = [] groups = [r for r in results if isinstance(r, ContactGroup)] group_counts = ContactGroupCount.get_totals(groups) if groups else {} for obj in results: if isinstance(obj, ContactGroup): result = {"id": "g-%s" % obj.uuid, "text": obj.name, "extra": group_counts[obj]} elif isinstance(obj, Contact): if org.is_anon: result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org)} else: result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org), "extra": obj.get_urn_display()} elif isinstance(obj, ContactURN): result = { "id": "u-%d" % obj.id, "text": obj.get_display(org), "extra": obj.contact.name or None, "scheme": obj.scheme, } formatted.append(result) return formatted
def count(self): if self.is_es_search(): # execute search on the ElasticSearch to get the count return self.object_list.count() else: # get the group count from the ContactGroupCount squashed model group_instance = self.object_list._hints.get("instance") if group_instance: return ContactGroupCount.get_totals([group_instance]).get(group_instance) else: return 0
def count(self): if isinstance(self.object_list, ModelESSearch): # execute search on the ElasticSearch to get the count return self.object_list.count() else: # get the group count from the ContactGroupCount squashed model group_instance = self.object_list._hints.get("instance") if group_instance: return ContactGroupCount.get_totals([group_instance]).get(group_instance) else: return 0
def create_contacts(self, orgs, locations, num_contacts): """ Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn id to avoid trying to hold all contact and URN objects in memory. """ group_counts = defaultdict(int) self._log("Creating %d test contacts..." % (len(orgs) * len(USERS))) for org in orgs: test_contacts = [] for user in org.cache['users']: test_contacts.append(Contact.get_test_contact(user)) org.cache['test_contacts'] = test_contacts self._log(self.style.SUCCESS("OK") + '\n') self._log("Creating %d regular contacts...\n" % num_contacts) # disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed # count row for every contact with DisableTriggersOn(Contact, ContactURN, Value, ContactGroup.contacts.through): names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]] names = [n if n else None for n in names] batch_num = 1 for index_batch in chunk_list(six.moves.xrange(num_contacts), self.batch_size): batch = [] # generate flat representations and contact objects for this batch for c_index in index_batch: # pragma: no cover org = self.random_org(orgs) name = self.random_choice(names) location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None created_on = self.timeline_date(c_index / num_contacts) c = { 'org': org, 'user': org.cache['users'][0], 'name': name, 'groups': [], 'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None, 'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None, 'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'ward': location[0] if location else None, 'district': location[1] if location else None, 'state': location[2] if location else None, 'language': self.random_choice(CONTACT_LANGS), 'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB), 'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB), 'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB), 'created_on': created_on, 'modified_on': self.random_date(created_on, self.db_ends_on), } # work out which system groups this contact belongs to if c['is_active']: if not c['is_blocked'] and not c['is_stopped']: c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_ALL]) if c['is_blocked']: c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED]) if c['is_stopped']: c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_STOPPED]) # let each user group decide if it is taking this contact for g in org.cache['groups']: if g.member(c) if callable(g.member) else self.probability(g.member): c['groups'].append(g) # track changes to group counts for g in c['groups']: group_counts[g] += 1 batch.append(c) self._create_contact_batch(batch) self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1))) batch_num += 1 # create group count records manually counts = [] for group, count in group_counts.items(): counts.append(ContactGroupCount(group=group, count=count, is_squashed=True)) group.count = count ContactGroupCount.objects.bulk_create(counts)
def create_contacts(self, orgs, locations, num_contacts): """ Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn id to avoid trying to hold all contact and URN objects in memory. """ group_counts = defaultdict(int) self._log(self.style.SUCCESS("OK") + "\n") self._log("Creating %d regular contacts...\n" % num_contacts) # disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed # count row for every contact with DisableTriggersOn(Contact, ContactURN, ContactGroup.contacts.through): names = [("%s %s" % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]] names = [n if n else None for n in names] batch_num = 1 for index_batch in chunk_list(range(num_contacts), self.batch_size): batch = [] # generate flat representations and contact objects for this batch for c_index in index_batch: # pragma: no cover org = self.random_org(orgs) name = self.random_choice(names) location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None created_on = self.timeline_date(c_index / num_contacts) c = { "org": org, "user": org.cache["users"][0], "name": name, "groups": [], "tel": "+2507%08d" % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None, "twitter": "%s%d" % (name.replace(" ", "_").lower() if name else "tweep", c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None, "gender": self.random_choice(("M", "F")) if self.probability(CONTACT_HAS_FIELD_PROB) else None, "age": self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None, "joined": self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None, "ward": location[0] if location else None, "district": location[1] if location else None, "state": location[2] if location else None, "language": self.random_choice(CONTACT_LANGS), "is_stopped": self.probability(CONTACT_IS_STOPPED_PROB), "is_blocked": self.probability(CONTACT_IS_BLOCKED_PROB), "is_active": self.probability(1 - CONTACT_IS_DELETED_PROB), "created_on": created_on, "modified_on": self.random_date(created_on, self.db_ends_on), "fields_as_json": {}, } if c["gender"] is not None: c["fields_as_json"][str(org.cache["fields"]["gender"].uuid)] = {"text": str(c["gender"])} if c["age"] is not None: c["fields_as_json"][str(org.cache["fields"]["age"].uuid)] = { "text": str(c["age"]), "number": str(c["age"]), } if c["joined"] is not None: c["fields_as_json"][str(org.cache["fields"]["joined"].uuid)] = { "text": org.format_datetime(c["joined"], show_time=False), "datetime": timezone.localtime(c["joined"], org.timezone).isoformat(), } if location: c["fields_as_json"].update( { str(org.cache["fields"]["ward"].uuid): { "text": str(c["ward"].path.split(" > ")[-1]), "ward": c["ward"].path, "district": c["district"].path, "state": c["state"].path, }, str(org.cache["fields"]["district"].uuid): { "text": str(c["district"].path.split(" > ")[-1]), "district": c["district"].path, "state": c["state"].path, }, str(org.cache["fields"]["state"].uuid): { "text": str(c["state"].path.split(" > ")[-1]), "state": c["state"].path, }, } ) # work out which system groups this contact belongs to if c["is_active"]: if not c["is_blocked"] and not c["is_stopped"]: c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_ALL]) if c["is_blocked"]: c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_BLOCKED]) if c["is_stopped"]: c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_STOPPED]) # let each user group decide if it is taking this contact for g in org.cache["groups"]: if g.member(c) if callable(g.member) else self.probability(g.member): c["groups"].append(g) # track changes to group counts for g in c["groups"]: group_counts[g] += 1 batch.append(c) self._create_contact_batch(batch) self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1))) batch_num += 1 # create group count records manually counts = [] for group, count in group_counts.items(): counts.append(ContactGroupCount(group=group, count=count, is_squashed=True)) group.count = count ContactGroupCount.objects.bulk_create(counts)
def populate_contactgroupcounts(apps, schema_editor): from temba.contacts.models import ContactGroup, ContactGroupCount for group_id in ContactGroup.all_groups.all().values_list('id', flat=True): group = ContactGroup.all_groups.get(id=group_id) count = ContactGroupCount.populate_for_group(group) print "%s %d" % (group.name, count.count)
def create_contacts(self, orgs, locations, num_total): batch_size = 5000 num_test_contacts = len(orgs) * len(USERS) group_membership_model = ContactGroup.contacts.through group_counts = defaultdict(int) self._log("Creating %d test contacts...\n" % num_test_contacts) for org in orgs: for user in org.cache['users']: Contact.get_test_contact(user) self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts)) base_contact_id = self.get_current_id(Contact) + 1 # Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed # count row for every contact with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model): names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]] names = [n if n else None for n in names] batch = 1 for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size): contacts = [] urns = [] values = [] memberships = [] def add_to_group(g): group_counts[g] += 1 memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g)) for c_index in index_batch: # pragma: no cover org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org name = self.random_choice(names) location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total) c = { 'id': base_contact_id + c_index, # database id this contact will have when created 'org': org, 'user': org.cache['users'][0], 'name': name, 'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None, 'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None, 'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None, 'ward': location[0] if location else None, 'district': location[1] if location else None, 'state': location[2] if location else None, 'language': self.random_choice(CONTACT_LANGS), 'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB), 'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB), 'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB), 'created_on': created_on, 'modified_on': self.random_date(created_on, self.db_ends_on), } if c['is_active']: if not c['is_blocked'] and not c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL]) if c['is_blocked']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED]) if c['is_stopped']: add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED]) contacts.append(Contact(org=org, name=c['name'], language=c['language'], is_stopped=c['is_stopped'], is_blocked=c['is_blocked'], is_active=c['is_active'], created_by=user, created_on=c['created_on'], modified_by=user, modified_on=c['modified_on'])) if c['tel']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME, path=c['tel'], urn=URN.from_tel(c['tel']))) if c['twitter']: urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME, path=c['twitter'], urn=URN.from_twitter(c['twitter']))) if c['gender']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'], string_value=c['gender'])) if c['age']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'], string_value=str(c['age']), decimal_value=c['age'])) if c['joined']: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'], string_value=datetime_to_str(c['joined']), datetime_value=c['joined'])) if location: values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'], string_value=c['ward'].name, location_value=c['ward'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'], string_value=c['district'].name, location_value=c['district'])) values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'], string_value=c['state'].name, location_value=c['state'])) # let each group decide if it is taking this contact for g in org.cache['groups']: if g.member(c) if callable(g.member) else self.probability(g.member): add_to_group(g) Contact.objects.bulk_create(contacts) ContactURN.objects.bulk_create(urns) Value.objects.bulk_create(values) group_membership_model.objects.bulk_create(memberships) self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1))) batch += 1 # create group count records manually counts = [] for group, count in group_counts.items(): counts.append(ContactGroupCount(group=group, count=count, is_squashed=True)) ContactGroupCount.objects.bulk_create(counts) # for sanity check that our presumed last contact id matches the last actual contact id assert c['id'] == Contact.objects.order_by('-id').first().id