def omnibox_results_to_dict(org, results):
"""
Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text}
"""
formatted = []
groups = [r for r in results if isinstance(r, ContactGroup)]
group_counts = ContactGroupCount.get_totals(groups) if groups else {}
for obj in results:
if isinstance(obj, ContactGroup):
result = {
'id': 'g-%s' % obj.uuid,
'text': obj.name,
'extra': group_counts[obj]
}
elif isinstance(obj, Contact):
result = {
'id': 'c-%s' % obj.uuid,
'text': obj.get_display(org)
}
elif isinstance(obj, ContactURN):
result = {
'id': 'u-%d' % obj.id,
'text': obj.get_display(org),
'extra': obj.contact.name or None,
'scheme': obj.scheme
}
formatted.append(result)
return formatted
def omnibox_results_to_dict(org, results):
"""
Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text}
"""
formatted = []
groups = [r for r in results if isinstance(r, ContactGroup)]
group_counts = ContactGroupCount.get_totals(groups) if groups else {}
for obj in results:
if isinstance(obj, ContactGroup):
result = {"id": "g-%s" % obj.uuid, "text": obj.name, "extra": group_counts[obj]}
elif isinstance(obj, Contact):
if org.is_anon:
result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org)}
else:
result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org), "extra": obj.get_urn_display()}
elif isinstance(obj, ContactURN):
result = {
"id": "u-%d" % obj.id,
"text": obj.get_display(org),
"extra": obj.contact.name or None,
"scheme": obj.scheme,
}
formatted.append(result)
return formatted
def populate_contactgroupcounts(apps, schema_editor):
from temba.contacts.models import ContactGroup, ContactGroupCount
for group_id in ContactGroup.all_groups.all().values_list('id',
flat=True):
group = ContactGroup.all_groups.get(id=group_id)
count = ContactGroupCount.populate_for_group(group)
print "%s %d" % (group.name, count.count)
def omnibox_results_to_dict(org, results):
"""
Converts the result of a omnibox query (queryset of contacts, groups or URNs, or a list) into a dict {id, text}
"""
formatted = []
groups = [r for r in results if isinstance(r, ContactGroup)]
group_counts = ContactGroupCount.get_totals(groups) if groups else {}
for obj in results:
if isinstance(obj, ContactGroup):
result = {"id": "g-%s" % obj.uuid, "text": obj.name, "extra": group_counts[obj]}
elif isinstance(obj, Contact):
if org.is_anon:
result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org)}
else:
result = {"id": "c-%s" % obj.uuid, "text": obj.get_display(org), "extra": obj.get_urn_display()}
elif isinstance(obj, ContactURN):
result = {
"id": "u-%d" % obj.id,
"text": obj.get_display(org),
"extra": obj.contact.name or None,
"scheme": obj.scheme,
}
formatted.append(result)
return formatted
def count(self):
if self.is_es_search():
# execute search on the ElasticSearch to get the count
return self.object_list.count()
else:
# get the group count from the ContactGroupCount squashed model
group_instance = self.object_list._hints.get("instance")
if group_instance:
return ContactGroupCount.get_totals([group_instance]).get(group_instance)
else:
return 0
def count(self):
if isinstance(self.object_list, ModelESSearch):
# execute search on the ElasticSearch to get the count
return self.object_list.count()
else:
# get the group count from the ContactGroupCount squashed model
group_instance = self.object_list._hints.get("instance")
if group_instance:
return ContactGroupCount.get_totals([group_instance]).get(group_instance)
else:
return 0
def create_contacts(self, orgs, locations, num_contacts):
"""
Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn
id to avoid trying to hold all contact and URN objects in memory.
"""
group_counts = defaultdict(int)
self._log("Creating %d test contacts..." % (len(orgs) * len(USERS)))
for org in orgs:
test_contacts = []
for user in org.cache['users']:
test_contacts.append(Contact.get_test_contact(user))
org.cache['test_contacts'] = test_contacts
self._log(self.style.SUCCESS("OK") + '\n')
self._log("Creating %d regular contacts...\n" % num_contacts)
# disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, ContactGroup.contacts.through):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch_num = 1
for index_batch in chunk_list(six.moves.xrange(num_contacts), self.batch_size):
batch = []
# generate flat representations and contact objects for this batch
for c_index in index_batch: # pragma: no cover
org = self.random_org(orgs)
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(c_index / num_contacts)
c = {
'org': org,
'user': org.cache['users'][0],
'name': name,
'groups': [],
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
# work out which system groups this contact belongs to
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
c['groups'].append(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
# let each user group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
c['groups'].append(g)
# track changes to group counts
for g in c['groups']:
group_counts[g] += 1
batch.append(c)
self._create_contact_batch(batch)
self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1)))
batch_num += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
group.count = count
ContactGroupCount.objects.bulk_create(counts)
def create_contacts(self, orgs, locations, num_contacts):
"""
Creates test and regular contacts for this database. Returns tuples of org, contact id and the preferred urn
id to avoid trying to hold all contact and URN objects in memory.
"""
group_counts = defaultdict(int)
self._log(self.style.SUCCESS("OK") + "\n")
self._log("Creating %d regular contacts...\n" % num_contacts)
# disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, ContactGroup.contacts.through):
names = [("%s %s" % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch_num = 1
for index_batch in chunk_list(range(num_contacts), self.batch_size):
batch = []
# generate flat representations and contact objects for this batch
for c_index in index_batch: # pragma: no cover
org = self.random_org(orgs)
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(c_index / num_contacts)
c = {
"org": org,
"user": org.cache["users"][0],
"name": name,
"groups": [],
"tel": "+2507%08d" % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
"twitter": "%s%d" % (name.replace(" ", "_").lower() if name else "tweep", c_index)
if self.probability(CONTACT_HAS_TWITTER_PROB)
else None,
"gender": self.random_choice(("M", "F")) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"age": self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"joined": self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
"ward": location[0] if location else None,
"district": location[1] if location else None,
"state": location[2] if location else None,
"language": self.random_choice(CONTACT_LANGS),
"is_stopped": self.probability(CONTACT_IS_STOPPED_PROB),
"is_blocked": self.probability(CONTACT_IS_BLOCKED_PROB),
"is_active": self.probability(1 - CONTACT_IS_DELETED_PROB),
"created_on": created_on,
"modified_on": self.random_date(created_on, self.db_ends_on),
"fields_as_json": {},
}
if c["gender"] is not None:
c["fields_as_json"][str(org.cache["fields"]["gender"].uuid)] = {"text": str(c["gender"])}
if c["age"] is not None:
c["fields_as_json"][str(org.cache["fields"]["age"].uuid)] = {
"text": str(c["age"]),
"number": str(c["age"]),
}
if c["joined"] is not None:
c["fields_as_json"][str(org.cache["fields"]["joined"].uuid)] = {
"text": org.format_datetime(c["joined"], show_time=False),
"datetime": timezone.localtime(c["joined"], org.timezone).isoformat(),
}
if location:
c["fields_as_json"].update(
{
str(org.cache["fields"]["ward"].uuid): {
"text": str(c["ward"].path.split(" > ")[-1]),
"ward": c["ward"].path,
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["district"].uuid): {
"text": str(c["district"].path.split(" > ")[-1]),
"district": c["district"].path,
"state": c["state"].path,
},
str(org.cache["fields"]["state"].uuid): {
"text": str(c["state"].path.split(" > ")[-1]),
"state": c["state"].path,
},
}
)
# work out which system groups this contact belongs to
if c["is_active"]:
if not c["is_blocked"] and not c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_ALL])
if c["is_blocked"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_BLOCKED])
if c["is_stopped"]:
c["groups"].append(org.cache["system_groups"][ContactGroup.TYPE_STOPPED])
# let each user group decide if it is taking this contact
for g in org.cache["groups"]:
if g.member(c) if callable(g.member) else self.probability(g.member):
c["groups"].append(g)
# track changes to group counts
for g in c["groups"]:
group_counts[g] += 1
batch.append(c)
self._create_contact_batch(batch)
self._log(" > Created batch %d of %d\n" % (batch_num, max(num_contacts // self.batch_size, 1)))
batch_num += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
group.count = count
ContactGroupCount.objects.bulk_create(counts)
def populate_contactgroupcounts(apps, schema_editor):
from temba.contacts.models import ContactGroup, ContactGroupCount
for group_id in ContactGroup.all_groups.all().values_list('id', flat=True):
group = ContactGroup.all_groups.get(id=group_id)
count = ContactGroupCount.populate_for_group(group)
print "%s %d" % (group.name, count.count)
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
