def test_squash_reporters(self): with self.settings(CACHES={'default': {'BACKEND': 'redis_cache.cache.RedisCache', 'LOCATION': '127.0.0.1:6379:1', 'OPTIONS': {'CLIENT_CLASS': 'redis_cache.client.DefaultClient'} }}): self.assertFalse(ReportersCounter.objects.all()) counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2) counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1) counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3) self.assertEqual(ReportersCounter.objects.all().count(), 3) self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2) ReportersCounter.squash_counts() self.assertEqual(ReportersCounter.objects.all().count(), 2) # type-a counters are squashed into one row self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk])) self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1) self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk)) counter_type_a = ReportersCounter.objects.filter(type='type-a').first() self.assertTrue(counter_type_a.count, 5)
def test_squash_reporters(self, mock_redis_get): mock_redis_get.return_value = None self.assertFalse(ReportersCounter.objects.all()) counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2) counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1) counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3) self.assertEqual(ReportersCounter.objects.all().count(), 3) self.assertEqual( ReportersCounter.objects.filter(type='type-a').count(), 2) ReportersCounter.squash_counts() self.assertEqual(ReportersCounter.objects.all().count(), 2) # type-a counters are squashed into one row self.assertFalse( ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk])) self.assertEqual( ReportersCounter.objects.filter(type='type-a').count(), 1) self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk)) counter_type_a = ReportersCounter.objects.filter(type='type-a').first() self.assertTrue(counter_type_a.count, 5)
def pull_contacts(org, since, until): """ Fetches updated contacts from RapidPro and updates local contacts accordingly """ from ureport.backend import get_backend from ureport.contacts.models import ReportersCounter backend = get_backend() if not since: logger.warn("First time run for org #%d. Will sync all contacts" % org.pk) start = time.time() fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org) logger.warn("Fetched contact fields for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, fields_created, fields_updated, fields_deleted, ignored)) logger.warn("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start)) start_boundaries = time.time() boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org) logger.warn("Fetched boundaries for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, boundaries_created, boundaries_updated, boundaries_deleted, ignored)) logger.warn("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries)) start_contacts = time.time() contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until) logger.warn("Fetched contacts for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, contacts_created, contacts_updated, contacts_deleted, ignored)) logger.warn("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts)) # Squash reporters counts ReportersCounter.squash_counts() return { 'fields': {'created': fields_created, 'updated': fields_updated, 'deleted': fields_deleted}, 'boundaries': {'created': boundaries_created, 'updated': boundaries_updated, 'deleted': boundaries_deleted}, 'contacts': {'created': contacts_created, 'updated': contacts_updated, 'deleted': contacts_deleted} }
def test_squash_reporters(self): with self.settings( CACHES={ 'default': { 'BACKEND': 'redis_cache.cache.RedisCache', 'LOCATION': '127.0.0.1:6379:1', 'OPTIONS': { 'CLIENT_CLASS': 'redis_cache.client.DefaultClient' } } }): self.assertFalse(ReportersCounter.objects.all()) counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2) counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1) counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3) self.assertEqual(ReportersCounter.objects.all().count(), 3) self.assertEqual( ReportersCounter.objects.filter(type='type-a').count(), 2) ReportersCounter.squash_counts() self.assertEqual(ReportersCounter.objects.all().count(), 2) # type-a counters are squashed into one row self.assertFalse( ReportersCounter.objects.filter( pk__in=[counter1.pk, counter3.pk])) self.assertEqual( ReportersCounter.objects.filter(type='type-a').count(), 1) self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk)) counter_type_a = ReportersCounter.objects.filter( type='type-a').first() self.assertTrue(counter_type_a.count, 5)
def test_squash_reporters(self, mock_redis_get): mock_redis_get.return_value = None self.assertFalse(ReportersCounter.objects.all()) counter1 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=2) counter2 = ReportersCounter.objects.create(org=self.nigeria, type='type-b', count=1) counter3 = ReportersCounter.objects.create(org=self.nigeria, type='type-a', count=3) self.assertEqual(ReportersCounter.objects.all().count(), 3) self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 2) ReportersCounter.squash_counts() self.assertEqual(ReportersCounter.objects.all().count(), 2) # type-a counters are squashed into one row self.assertFalse(ReportersCounter.objects.filter(pk__in=[counter1.pk, counter3.pk])) self.assertEqual(ReportersCounter.objects.filter(type='type-a').count(), 1) self.assertTrue(ReportersCounter.objects.filter(pk=counter2.pk)) counter_type_a = ReportersCounter.objects.filter(type='type-a').first() self.assertTrue(counter_type_a.count, 5)
def pull_contacts(org, ignored_since, ignored_until): """ Fetches updated contacts from RapidPro and updates local contacts accordingly """ from ureport.contacts.models import ReportersCounter results = dict() backends = org.backends.filter(is_active=True) for backend_obj in backends: backend = org.get_backend(backend_slug=backend_obj.slug) last_fetch_date_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % (org.pk, backend_obj.slug) until = datetime_to_json_date(timezone.now()) since = cache.get(last_fetch_date_key, None) if not since: logger.info("First time run for org #%d. Will sync all contacts" % org.pk) start = time.time() fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields(org) logger.info( "Fetched contact fields for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, fields_created, fields_updated, fields_deleted, ignored) ) logger.info("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start)) start_boundaries = time.time() boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries(org) logger.info( "Fetched boundaries for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, boundaries_created, boundaries_updated, boundaries_deleted, ignored) ) logger.info("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries)) start_contacts = time.time() contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts(org, since, until) cache.set(last_fetch_date_key, until, None) logger.info( "Fetched contacts for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, contacts_created, contacts_updated, contacts_deleted, ignored) ) logger.info("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts)) # Squash reporters counts ReportersCounter.squash_counts() results[backend_obj.slug] = { "fields": {"created": fields_created, "updated": fields_updated, "deleted": fields_deleted}, "boundaries": { "created": boundaries_created, "updated": boundaries_updated, "deleted": boundaries_deleted, }, "contacts": {"created": contacts_created, "updated": contacts_updated, "deleted": contacts_deleted}, } return results
def pull_contacts(org, since, until): """ Fetches updated contacts from RapidPro and updates local contacts accordingly """ from ureport.backend import get_backend from ureport.contacts.models import ReportersCounter backend = get_backend() if not since: logger.warn("First time run for org #%d. Will sync all contacts" % org.pk) start = time.time() fields_created, fields_updated, fields_deleted, ignored = backend.pull_fields( org) logger.warn( "Fetched contact fields for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, fields_created, fields_updated, fields_deleted, ignored)) logger.warn("Fetch fields for org #%d took %ss" % (org.pk, time.time() - start)) start_boundaries = time.time() boundaries_created, boundaries_updated, boundaries_deleted, ignored = backend.pull_boundaries( org) logger.warn("Fetched boundaries for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, boundaries_created, boundaries_updated, boundaries_deleted, ignored)) logger.warn("Fetch boundaries for org #%d took %ss" % (org.pk, time.time() - start_boundaries)) start_contacts = time.time() contacts_created, contacts_updated, contacts_deleted, ignored = backend.pull_contacts( org, since, until) logger.warn("Fetched contacts for org #%d. " "Created %s, Updated %s, Deleted %d, Ignored %d" % (org.pk, contacts_created, contacts_updated, contacts_deleted, ignored)) logger.warn("Fetch contacts for org #%d took %ss" % (org.pk, time.time() - start_contacts)) # Squash reporters counts ReportersCounter.squash_counts() return { 'fields': { 'created': fields_created, 'updated': fields_updated, 'deleted': fields_deleted }, 'boundaries': { 'created': boundaries_created, 'updated': boundaries_updated, 'deleted': boundaries_deleted }, 'contacts': { 'created': contacts_created, 'updated': contacts_updated, 'deleted': contacts_deleted } }