def test_squash_reporters(self):
with self.settings(CACHES={'default': {'BACKEND': 'redis_cache.cache.RedisCache',
'LOCATION': '127.0.0.1:6379:1',
'OPTIONS': {'CLIENT_CLASS': 'redis_cache.client.DefaultClient'}
}}):
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def test_squash_reporters(self, mock_redis_get):
mock_redis_get.return_value = None
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria,
type='type-b',
count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(
ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def pull_contacts(org, since, until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.backend import get_backend
from ureport.contacts.models import ReportersCounter
backend = get_backend()
if not since:
logger.warn("First time run for org #%d. Will sync all contacts" % org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org)
logger.warn("Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, fields_created, fields_updated,
fields_deleted, ignored))
logger.warn("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org)
logger.warn("Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, boundaries_created, boundaries_updated,
boundaries_deleted, ignored))
logger.warn("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until)
logger.warn("Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, contacts_created, contacts_updated,
contacts_deleted, ignored))
logger.warn("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
return {
'fields': {'created': fields_created, 'updated': fields_updated, 'deleted': fields_deleted},
'boundaries': {'created': boundaries_created, 'updated': boundaries_updated, 'deleted': boundaries_deleted},
'contacts': {'created': contacts_created, 'updated': contacts_updated, 'deleted': contacts_deleted}
}
def test_squash_reporters(self):
with self.settings(
CACHES={
'default': {
'BACKEND': 'redis_cache.cache.RedisCache',
'LOCATION': '127.0.0.1:6379:1',
'OPTIONS': {
'CLIENT_CLASS': 'redis_cache.client.DefaultClient'
}
}
}):
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria,
type='type-b',
count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria,
type='type-a',
count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(
ReportersCounter.objects.filter(
pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(
ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(
type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def test_squash_reporters(self, mock_redis_get):
mock_redis_get.return_value = None
self.assertFalse(ReportersCounter.objects.all())
counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2)
counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1)
counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3)
self.assertEqual(ReportersCounter.objects.all().count(), 3)
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2)
ReportersCounter.squash_counts()
self.assertEqual(ReportersCounter.objects.all().count(), 2)
# type-a counters are squashed into one row
self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk]))
self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1)
self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk))
counter_type_a = ReportersCounter.objects.filter(type='type-a').first()
self.assertTrue(counter_type_a.count, 5)
def pull_contacts(org, ignored_since, ignored_until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.contacts.models import ReportersCounter
results = dict()
backends = org.backends.filter(is_active=True)
for backend_obj in backends:
backend = org.get_backend(backend_slug=backend_obj.slug)
last_fetch_date_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % (org.pk, backend_obj.slug)
until = datetime_to_json_date(timezone.now())
since = cache.get(last_fetch_date_key, None)
if not since:
logger.info("First time run for org #%d. Will sync all contacts" % org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org)
logger.info(
"Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, fields_created, fields_updated, fields_deleted, ignored)
)
logger.info("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org)
logger.info(
"Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, boundaries_created, boundaries_updated, boundaries_deleted, ignored)
)
logger.info("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until)
cache.set(last_fetch_date_key, until, None)
logger.info(
"Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d"
% (org.pk, contacts_created, contacts_updated, contacts_deleted, ignored)
)
logger.info("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
results[backend_obj.slug] = {
"fields": {"created": fields_created, "updated": fields_updated, "deleted": fields_deleted},
"boundaries": {
"created": boundaries_created,
"updated": boundaries_updated,
"deleted": boundaries_deleted,
},
"contacts": {"created": contacts_created, "updated": contacts_updated, "deleted": contacts_deleted},
}
return results
def pull_contacts(org, since, until):
"""
Fetches updated contacts from RapidPro and updates local contacts accordingly
"""
from ureport.backend import get_backend
from ureport.contacts.models import ReportersCounter
backend = get_backend()
if not since:
logger.warn("First time run for org #%d. Will sync all contacts" %
org.pk)
start = time.time()
fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(
org)
logger.warn(
"Fetched contact fields for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, fields_created, fields_updated, fields_deleted, ignored))
logger.warn("Fetch fields for org #%d took %ss" %
(org.pk, time.time() - start))
start_boundaries = time.time()
boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(
org)
logger.warn("Fetched boundaries for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, boundaries_created, boundaries_updated,
boundaries_deleted, ignored))
logger.warn("Fetch boundaries for org #%d took %ss" %
(org.pk, time.time() - start_boundaries))
start_contacts = time.time()
contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(
org, since, until)
logger.warn("Fetched contacts for org #%d. "
"Created %s, Updated %s, Deleted %d, Ignored %d" %
(org.pk, contacts_created, contacts_updated, contacts_deleted,
ignored))
logger.warn("Fetch contacts for org #%d took %ss" %
(org.pk, time.time() - start_contacts))
# Squash reporters counts
ReportersCounter.squash_counts()
return {
'fields': {
'created': fields_created,
'updated': fields_updated,
'deleted': fields_deleted
},
'boundaries': {
'created': boundaries_created,
'updated': boundaries_updated,
'deleted': boundaries_deleted
},
'contacts': {
'created': contacts_created,
'updated': contacts_updated,
'deleted': contacts_deleted
}
}
