def test_contact_ward_field(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid="C-007",
org=self.nigeria,
gender="M",
born=1990,
occupation="Student",
registered_on=json_date_to_datetime("2014-01-02T03:04:05.000"),
state="R-LAGOS",
district="R-OYO",
ward="R-IKEJA",
)
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count["ward:R-IKEJA"], 1)
Contact.objects.create(
uuid="C-008",
org=self.nigeria,
gender="M",
born=1980,
occupation="Teacher",
registered_on=json_date_to_datetime("2014-01-02T03:07:05.000"),
state="R-LAGOS",
district="R-OYO",
ward="R-IKEJA",
)
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count["ward:R-IKEJA"], 2)
Contact.objects.all().delete()
def test_squash_reporters(self, mock_redis_get):
mock_redis_get.return_value = None
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria,
type='type-b',
count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(
ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def test_contact_ward_field(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid="C-007",
org=self.nigeria,
gender="M",
born=1990,
occupation="Student",
registered_on=json_date_to_datetime("2014-01-02T03:04:05.000"),
state="R-LAGOS",
district="R-OYO",
ward="R-IKEJA",
)
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count["ward:R-IKEJA"], 1)
Contact.objects.create(
uuid="C-008",
org=self.nigeria,
gender="M",
born=1980,
occupation="Teacher",
registered_on=json_date_to_datetime("2014-01-02T03:07:05.000"),
state="R-LAGOS",
district="R-OYO",
ward="R-IKEJA",
)
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count["ward:R-IKEJA"], 2)
Contact.objects.all().delete()
def test_reporters_counter(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(uuid='C-007', org=self.nigeria, gender='M', born=1990, occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'), state='R-LAGOS',
district='R-OYO')
expected = dict()
expected['total-reporters'] = 1
expected['gender:m'] = 1
expected['occupation:student'] = 1
expected['born:1990'] = 1
expected['registered_on:2014-01-02'] = 1
expected['state:R-LAGOS'] = 1
expected['district:R-OYO'] = 1
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
Contact.objects.create(uuid='C-008', org=self.nigeria, gender='M', born=1980, occupation='Teacher',
registered_on=json_date_to_datetime('2014-01-02T03:07:05.000'), state='R-LAGOS',
district='R-OYO')
expected = dict()
expected['total-reporters'] = 2
expected['gender:m'] = 2
expected['occupation:student'] = 1
expected['occupation:teacher'] = 1
expected['born:1990'] = 1
expected['born:1980'] = 1
expected['registered_on:2014-01-02'] = 2
expected['state:R-LAGOS'] = 2
expected['district:R-OYO'] = 2
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
def test_contact_ward_field(self):
temba_contact = TembaContact.create(uuid='C-0011', name="Jan", urns=['tel:123'],
groups=['G-001', 'G-007'],
fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos',
'lga': '', 'ward': 'Ikeja', 'occupation': 'Student', 'born': '1990',
'gender': 'Male'},
language='eng')
kwargs = Contact.kwargs_from_temba(self.nigeria, temba_contact)
# invalid parent boundary (district) will yield empty ward
self.assertEqual(kwargs, dict(uuid='C-0011', org=self.nigeria, gender='M', born=1990, occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'), state='R-LAGOS',
district='', ward=''))
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(uuid='C-007', org=self.nigeria, gender='M', born=1990, occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'), state='R-LAGOS',
district='R-OYO', ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 1)
Contact.objects.create(uuid='C-008', org=self.nigeria, gender='M', born=1980, occupation='Teacher',
registered_on=json_date_to_datetime('2014-01-02T03:07:05.000'), state='R-LAGOS',
district='R-OYO', ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 2)
Contact.objects.all().delete()
def test_squash_reporters(self):
with self.settings(CACHES={'default': {'BACKEND': 'redis_cache.cache.RedisCache',
'LOCATION': '127.0.0.1:6379:1',
'OPTIONS': {'CLIENT_CLASS': 'redis_cache.client.DefaultClient'}
}}):
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def test_contact_ward_field(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid='C-007',
org=self.nigeria,
gender='M',
born=1990,
occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'),
state='R-LAGOS',
district='R-OYO',
ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 1)
Contact.objects.create(
uuid='C-008',
org=self.nigeria,
gender='M',
born=1980,
occupation='Teacher',
registered_on=json_date_to_datetime('2014-01-02T03:07:05.000'),
state='R-LAGOS',
district='R-OYO',
ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 2)
Contact.objects.all().delete()
def test_reporters_counter(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid='C-007',
org=self.nigeria,
gender='M',
born=1990,
occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'),
state='R-LAGOS',
district='R-OYO')
expected = dict()
expected['total-reporters'] = 1
expected['gender:m'] = 1
expected['occupation:student'] = 1
expected['born:1990'] = 1
expected['registered_on:2014-01-02'] = 1
expected['state:R-LAGOS'] = 1
expected['district:R-OYO'] = 1
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
Contact.objects.create(
uuid='C-008',
org=self.nigeria,
gender='M',
born=1980,
occupation='Teacher',
registered_on=json_date_to_datetime('2014-01-02T03:07:05.000'),
state='R-LAGOS',
district='R-OYO')
expected = dict()
expected['total-reporters'] = 2
expected['gender:m'] = 2
expected['occupation:student'] = 1
expected['occupation:teacher'] = 1
expected['born:1990'] = 1
expected['born:1980'] = 1
expected['registered_on:2014-01-02'] = 2
expected['state:R-LAGOS'] = 2
expected['district:R-OYO'] = 2
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
self.assertEqual(
ReportersCounter.get_counts(self.nigeria,
['total-reporters', 'gender:m']), {
'total-reporters': 2,
'gender:m': 2
})
def test_reporters_counter(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid="C-007",
org=self.nigeria,
gender="M",
born=1990,
occupation="Student",
registered_on=json_date_to_datetime("2014-01-02T03:04:05.000"),
state="R-LAGOS",
district="R-OYO",
)
expected = dict()
expected["total-reporters"] = 1
expected["gender:m"] = 1
expected["occupation:student"] = 1
expected["born:1990"] = 1
expected["registered_on:2014-01-02"] = 1
expected["state:R-LAGOS"] = 1
expected["district:R-OYO"] = 1
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
Contact.objects.create(
uuid="C-008",
org=self.nigeria,
gender="M",
born=1980,
occupation="Teacher",
registered_on=json_date_to_datetime("2014-01-02T03:07:05.000"),
state="R-LAGOS",
district="R-OYO",
)
expected = dict()
expected["total-reporters"] = 2
expected["gender:m"] = 2
expected["occupation:student"] = 1
expected["occupation:teacher"] = 1
expected["born:1990"] = 1
expected["born:1980"] = 1
expected["registered_on:2014-01-02"] = 2
expected["state:R-LAGOS"] = 2
expected["district:R-OYO"] = 2
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
self.assertEqual(
ReportersCounter.get_counts(self.nigeria, ["total-reporters", "gender:m"]),
{"total-reporters": 2, "gender:m": 2},
)
def update_cache_org_contact_counts(org):
from ureport.contacts.models import ReportersCounter
key = ORG_CONTACT_COUNT_KEY % org.pk
org_contacts_counts = ReportersCounter.get_counts(org)
cache.set(key, org_contacts_counts, ORG_CONTACT_COUNT_TIMEOUT)
return org_contacts_counts
def pull_contacts(org, since, until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.backend import get_backend
from ureport.contacts.models import ReportersCounter
backend = get_backend()
if not since:
logger.warn("First time run for org #%d. Will sync all contacts" % org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org)
logger.warn("Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, fields_created, fields_updated,
fields_deleted, ignored))
logger.warn("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org)
logger.warn("Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, boundaries_created, boundaries_updated,
boundaries_deleted, ignored))
logger.warn("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until)
logger.warn("Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, contacts_created, contacts_updated,
contacts_deleted, ignored))
logger.warn("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
return {
'fields': {'created': fields_created, 'updated': fields_updated, 'deleted': fields_deleted},
'boundaries': {'created': boundaries_created, 'updated': boundaries_updated, 'deleted': boundaries_deleted},
'contacts': {'created': contacts_created, 'updated': contacts_updated, 'deleted': contacts_deleted}
}
def test_squash_reporters(self):
with self.settings(
CACHES={
'default': {
'BACKEND': 'redis_cache.cache.RedisCache',
'LOCATION': '127.0.0.1:6379:1',
'OPTIONS': {
'CLIENT_CLASS': 'redis_cache.client.DefaultClient'
}
}
}):
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria,
type='type-b',
count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(
ReportersCounter.objects.filter(
pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(
type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def get_org_contacts_counts(org):
from ureport.contacts.models import ReportersCounter
key = ORG_CONTACT_COUNT_KEY % org.pk
org_contacts_counts = cache.get(key, None)
if org_contacts_counts:
return org_contacts_counts
org_contacts_counts = ReportersCounter.get_counts(org)
cache.set(key, org_contacts_counts, ORG_CONTACT_COUNT_TIMEOUT)
return org_contacts_counts
def test_squash_reporters(self, mock_redis_get):
mock_redis_get.return_value = None
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def test_reporters_counter(self):
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid="C-007",
org=self.nigeria,
gender="M",
born=1990,
occupation="Student",
registered_on=json_date_to_datetime("2014-01-02T03:04:05.000"),
state="R-LAGOS",
district="R-OYO",
)
expected = dict()
expected["total-reporters"] = 1
expected["gender:m"] = 1
expected["occupation:student"] = 1
expected["born:1990"] = 1
expected["registered_on:2014-01-02"] = 1
expected["registered_gender:2014-01-01:m"] = 1
expected["registered_born:2014-01-01:1990"] = 1
expected["registered_state:2014-01-01:R-LAGOS"] = 1
expected["state:R-LAGOS"] = 1
expected["district:R-OYO"] = 1
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
Contact.objects.create(
uuid="C-008",
org=self.nigeria,
gender="M",
born=1980,
occupation="Teacher",
registered_on=json_date_to_datetime("2014-01-02T03:07:05.000"),
state="R-LAGOS",
district="R-OYO",
)
expected = dict()
expected["total-reporters"] = 2
expected["gender:m"] = 2
expected["occupation:student"] = 1
expected["occupation:teacher"] = 1
expected["born:1990"] = 1
expected["born:1980"] = 1
expected["registered_on:2014-01-02"] = 2
expected["registered_gender:2014-01-01:m"] = 2
expected["registered_born:2014-01-01:1990"] = 1
expected["registered_born:2014-01-01:1980"] = 1
expected["registered_state:2014-01-01:R-LAGOS"] = 2
expected["state:R-LAGOS"] = 2
expected["district:R-OYO"] = 2
self.assertEqual(ReportersCounter.get_counts(self.nigeria), expected)
self.assertEqual(
ReportersCounter.get_counts(self.nigeria,
["total-reporters", "gender:m"]),
{
"total-reporters": 2,
"gender:m": 2
},
)
def test_contact_ward_field(self):
temba_contact = TembaContact.create(uuid='C-0011',
name="Jan",
urns=['tel:123'],
groups=['G-001', 'G-007'],
fields={
'registration_date':
'2014-01-02T03:04:05.000000Z',
'state': 'Lagos',
'lga': '',
'ward': 'Ikeja',
'occupation': 'Student',
'born': '1990',
'gender': 'Male'
},
language='eng')
kwargs = Contact.kwargs_from_temba(self.nigeria, temba_contact)
# invalid parent boundary (district) will yield empty ward
self.assertEqual(
kwargs,
dict(
uuid='C-0011',
org=self.nigeria,
gender='M',
born=1990,
occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'),
state='R-LAGOS',
district='',
ward=''))
self.assertEqual(ReportersCounter.get_counts(self.nigeria), dict())
Contact.objects.create(
uuid='C-007',
org=self.nigeria,
gender='M',
born=1990,
occupation='Student',
registered_on=json_date_to_datetime('2014-01-02T03:04:05.000'),
state='R-LAGOS',
district='R-OYO',
ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 1)
Contact.objects.create(
uuid='C-008',
org=self.nigeria,
gender='M',
born=1980,
occupation='Teacher',
registered_on=json_date_to_datetime('2014-01-02T03:07:05.000'),
state='R-LAGOS',
district='R-OYO',
ward='R-IKEJA')
field_count = ReportersCounter.get_counts(self.nigeria)
self.assertEqual(field_count['ward:R-IKEJA'], 2)
Contact.objects.all().delete()
def pull_contacts(org, since, until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.backend import get_backend
from ureport.contacts.models import ReportersCounter
backend = get_backend()
if not since:
logger.warn("First time run for org #%d. Will sync all contacts" %
org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(
org)
logger.warn(
"Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, fields_created, fields_updated, fields_deleted, ignored))
logger.warn("Fetch fields for org #%d took %ss" %
(org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(
org)
logger.warn("Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, boundaries_created, boundaries_updated,
boundaries_deleted, ignored))
logger.warn("Fetch boundaries for org #%d took %ss" %
(org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(
org, since, until)
logger.warn("Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, contacts_created, contacts_updated, contacts_deleted,
ignored))
logger.warn("Fetch contacts for org #%d took %ss" %
(org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
return {
'fields': {
'created': fields_created,
'updated': fields_updated,
'deleted': fields_deleted
},
'boundaries': {
'created': boundaries_created,
'updated': boundaries_updated,
'deleted': boundaries_deleted
},
'contacts': {
'created': contacts_created,
'updated': contacts_updated,
'deleted': contacts_deleted
}
}
def pull_contacts(org, ignored_since, ignored_until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.contacts.models import ReportersCounter
results = dict()
backends = org.backends.filter(is_active=True)
for backend_obj in backends:
backend = org.get_backend(backend_slug=backend_obj.slug)
last_fetch_date_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % (org.pk, backend_obj.slug)
until = datetime_to_json_date(timezone.now())
since = cache.get(last_fetch_date_key, None)
if not since:
logger.info("First time run for org #%d. Will sync all contacts" % org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org)
logger.info(
"Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, fields_created, fields_updated, fields_deleted, ignored)
)
logger.info("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org)
logger.info(
"Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, boundaries_created, boundaries_updated, boundaries_deleted, ignored)
)
logger.info("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until)
cache.set(last_fetch_date_key, until, None)
logger.info(
"Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, contacts_created, contacts_updated, contacts_deleted, ignored)
)
logger.info("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
results[backend_obj.slug] = {
"fields": {"created": fields_created, "updated": fields_updated, "deleted": fields_deleted},
"boundaries": {
"created": boundaries_created,
"updated": boundaries_updated,
"deleted": boundaries_deleted,
},
"contacts": {"created": contacts_created, "updated": contacts_updated, "deleted": contacts_deleted},
}
return results