def test_fetch_contact_messages(self, mock_get_messages): d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC) d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC) d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC) mock_get_messages.return_value = MockClientQuery([ TembaMessage.create( id=101, broadcast=201, contact=ObjectRef.create(uuid="C-001", name="Ann"), text="What is AIDS?", type='inbox', direction='in', visibility='visible', labels=[ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-111", name="Flagged")], created_on=d2 ) ]) messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3) self.assertEqual(messages, [{ 'id': 101, 'contact': {'uuid': "C-001", 'name': "Ann"}, 'text': "What is AIDS?", 'time': d2, 'labels': [{'id': self.aids.pk, 'name': "AIDS"}], 'flagged': True, 'archived': False, 'direction': 'I', 'flow': False, 'sender': None, 'broadcast': 201 }])
def test_update_required(self): d1 = now() - timedelta(hours=1) local = self.create_message(self.unicef, 101, self.ann, "Yes", [self.aids], is_flagged=False) # remote message has been flagged self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS"), ObjectRef.create(uuid='L-009', name="Flagged")], created_on=d1 ), {})) # remote message has been archived self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='archived', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS")], created_on=d1 ), {})) # remote message has been relabelled self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='archived', text="Yes", labels=[ObjectRef.create(uuid='L-002', name="Pregnancy")], created_on=d1 ), {})) # no differences self.assertFalse(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS")], created_on=d1 ), {}))
def test_update_required(self): d1 = now() - timedelta(hours=1) local = self.create_message(self.unicef, 101, self.ann, "Yes", [self.aids], is_flagged=False) # remote message has been flagged self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS"), ObjectRef.create(uuid='L-009', name="Flagged")], created_on=d1 ), {})) # remote message has been archived self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='archived', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS")], created_on=d1 ), {})) # remote message has been relabelled self.assertTrue(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='archived', text="Yes", labels=[ObjectRef.create(uuid='L-002', name="Pregnancy")], created_on=d1 ), {})) # no differences self.assertFalse(self.syncer.update_required(local, TembaMessage.create( id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="Yes", labels=[ObjectRef.create(uuid='L-001', name="AIDS")], created_on=d1 ), {}))
def test_local_kwargs(self): d1 = now() - timedelta(hours=1) d2 = now() - timedelta(days=32) remote = TembaMessage.create( id=123456789, contact=ObjectRef.create(uuid="C-001", name="Ann"), urn="twitter:ann123", direction="in", type="inbox", status="handled", visibility="visible", text="I have lots of questions!", labels=[ObjectRef.create(uuid="L-001", name="Spam"), ObjectRef.create(uuid="L-009", name="Flagged")], created_on=d1, ) kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual( kwargs, { "org": self.unicef, "backend_id": 123456789, "type": "I", "text": "I have lots of questions!", "is_flagged": True, "is_archived": False, "created_on": d1, "__data__contact": ("C-001", "Ann"), "__data__labels": [("L-001", "Spam")], }, ) # if remote is archived, so will local remote.visibility = "archived" kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual(kwargs["is_archived"], True) # if syncer is set to save as handled, local will be marked as handled kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote) self.assertTrue(kwargs["is_handled"], True) # if remote is too old, local will be marked as handled remote.created_on = d2 kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote) self.assertTrue(kwargs["is_handled"], True) # if remote is deleted, should return none remote.visibility = "deleted" self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_local_kwargs(self): d1 = now() - timedelta(hours=1) d2 = now() - timedelta(days=32) remote = TembaMessage.create( id=123456789, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="I have lots of questions!", labels=[ObjectRef.create(uuid='L-001', name="Spam"), ObjectRef.create(uuid='L-009', name="Flagged")], created_on=d1 ) kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual(kwargs, { 'org': self.unicef, 'backend_id': 123456789, 'type': 'I', 'text': "I have lots of questions!", 'is_flagged': True, 'is_archived': False, 'created_on': d1, '__data__contact': ("C-001", "Ann"), '__data__labels': [("L-001", "Spam")], }) # if remote is archived, so will local remote.visibility = 'archived' kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual(kwargs['is_archived'], True) # if syncer is set to save as handled, local will be marked as handled kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote) self.assertTrue(kwargs['is_handled'], True) # if remote is too old, local will be marked as handled remote.created_on = d2 kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote) self.assertTrue(kwargs['is_handled'], True) # if remote is deleted, should return none remote.visibility = 'deleted' self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_local_kwargs(self): d1 = now() - timedelta(hours=1) d2 = now() - timedelta(days=32) remote = TembaMessage.create( id=123456789, contact=ObjectRef.create(uuid='C-001', name="Ann"), urn="twitter:ann123", direction='in', type='inbox', status='handled', visibility='visible', text="I have lots of questions!", labels=[ObjectRef.create(uuid='L-001', name="Spam"), ObjectRef.create(uuid='L-009', name="Flagged")], created_on=d1 ) kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual(kwargs, { 'org': self.unicef, 'backend_id': 123456789, 'type': 'I', 'text': "I have lots of questions!", 'is_flagged': True, 'is_archived': False, 'created_on': d1, '__data__contact': ("C-001", "Ann"), '__data__labels': [("L-001", "Spam")], }) # if remote is archived, so will local remote.visibility = 'archived' kwargs = MessageSyncer().local_kwargs(self.unicef, remote) self.assertEqual(kwargs['is_archived'], True) # if syncer is set to save as handled, local will be marked as handled kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote) self.assertTrue(kwargs['is_handled'], True) # if remote is too old, local will be marked as handled remote.created_on = d2 kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote) self.assertTrue(kwargs['is_handled'], True) # if remote is deleted, should return none remote.visibility = 'deleted' self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_fetch_contact_messages(self, mock_get_messages): d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC) d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC) d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC) mock_get_messages.return_value = MockClientQuery([ TembaMessage.create( id=102, broadcast=201, contact=ObjectRef.create(uuid="C-001", name="Ann"), text="Welcome", type='inbox', direction='out', visibility='visible', labels=[], created_on=d3 ), TembaMessage.create( id=101, broadcast=None, contact=ObjectRef.create(uuid="C-001", name="Ann"), text="Hello", type='inbox', direction='in', visibility='archived', labels=[ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-111", name="Flagged")], created_on=d2 ) ]) messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3) self.assertEqual(messages, [ { 'id': 201, # id is the broadcast id 'contact': {'id': self.ann.pk, 'name': "Ann"}, 'urn': None, 'text': "Welcome", 'time': d3, 'direction': 'O', 'case': None, 'sender': None, } ]) # check that JSON schemas match local outgoing model outgoing = self.create_outgoing(self.unicef, self.admin, 201, 'B', "Hello", self.ann) self.assertEqual(messages[0].keys(), outgoing.as_json().keys())
def test_local_kwargs(self): kwargs = self.syncer.local_kwargs( self.unicef, TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={"age": "34"}, stopped=False, blocked=False, ), ) self.assertEqual( kwargs, { "org": self.unicef, "uuid": "C-001", "name": "Bob McFlow", "language": "eng", "is_blocked": False, "is_stopped": False, "is_stub": False, "fields": {"age": "34"}, "__data__groups": [("G-001", "Customers")], }, )
def test_fetch_contact_messages(self, mock_get_messages): d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC) d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC) d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC) mock_get_messages.return_value = MockClientQuery( [ TembaMessage.create( id=102, broadcast=201, contact=ObjectRef.create(uuid="C-001", name="Ann"), text="Welcome", type="inbox", direction="out", visibility="visible", labels=[], created_on=d3, ), TembaMessage.create( id=101, broadcast=None, contact=ObjectRef.create(uuid="C-001", name="Ann"), text="Hello", type="inbox", direction="in", visibility="archived", labels=[ ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-111", name="Flagged"), ], created_on=d2, ), ] ) messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3) self.assertEqual(len(messages), 1) self.assertIsInstance(messages[0], Outgoing) self.assertEqual(messages[0].backend_broadcast_id, 201) self.assertEqual(messages[0].contact, self.ann) self.assertEqual(messages[0].text, "Welcome") self.assertEqual(messages[0].created_on, d3)
def test_local_kwargs(self): kwargs = self.syncer.local_kwargs(self.unicef, TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={'age': "34"}, failed=False, blocked=False )) self.assertEqual(kwargs, { 'org': self.unicef, 'uuid': "C-001", 'name': "Bob McFlow", 'language': "eng", 'is_stub': False, 'fields': {'age': "34"}, '__data__groups': [("G-001", "Customers")], })
def test_local_kwargs(self): kwargs = self.syncer.local_kwargs(self.unicef, TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={'age': "34"}, stopped=False, blocked=False )) self.assertEqual(kwargs, { 'org': self.unicef, 'uuid': "C-001", 'name': "Bob McFlow", 'language': "eng", 'is_blocked': False, 'is_stopped': False, 'is_stub': False, 'fields': {'age': "34"}, '__data__groups': [("G-001", "Customers")], })
def test_pull_contacts(self, mock_get_contacts): Contact.objects.all().delete() # empty fetches mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery([]), # second call to get deleted contacts MockClientQuery([]) ] with self.assertNumQueries(0): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 0)) # fecthed contact not in configured group get ignored mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={'age': "34"}, failed=False, blocked=False ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': "67"}, failed=False, blocked=False ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={'age': "35"}, failed=True, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(4): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 3)) mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "34"}, failed=False, blocked=False ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': "67"}, failed=False, blocked=False ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={'age': "35"}, failed=True, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(8): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (1, 0, 0, 2)) Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "34"}, failed=False, blocked=False ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "67"}, failed=False, blocked=False ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={'age': "35"}, failed=True, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(9): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (2, 0, 0, 1)) Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts # all included in the reporters MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "34"}, failed=False, blocked=False ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "67"}, failed=False, blocked=False ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "35"}, failed=True, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) contact_jan = Contact.objects.filter(uuid='C-001').first() self.assertFalse(contact_jan.born) self.assertFalse(contact_jan.state) mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery( [ TembaContact.create(uuid='C-001', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos', 'lga': 'Oyo', 'occupation': 'Student', 'born': '1990', 'gender': 'Male'}, language='eng'), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={'age': "67", "born": "1992"}, failed=False, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(8): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 2, 0, 0)) contact_jan = Contact.objects.filter(uuid='C-001').first() self.assertTrue(contact_jan.born) self.assertEqual(contact_jan.born, 1990) self.assertTrue(contact_jan.state) self.assertEqual(contact_jan.state, 'R-LAGOS') self.assertTrue(Contact.objects.filter(uuid='C-002', is_active=True)) mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery([] ), # second call to get deleted contacts MockClientQuery( [ TembaContact.create( uuid="C-002", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(2): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 1, 0)) self.assertFalse(Contact.objects.filter(uuid='C-002', is_active=True))
def test_pull_contacts(self, mock_get_contacts): # start with nothing... Group.objects.all().delete() Field.objects.all().delete() Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={'age': "34"}, failed=False, blocked=False ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': "67"}, failed=False, blocked=False ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={'age': "35"}, failed=True, blocked=False ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(15): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) bob = Contact.objects.get(uuid="C-001") jim = Contact.objects.get(uuid="C-002") ann = Contact.objects.get(uuid="C-003") self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, jim, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), set()) # stub contact groups will have been created too customers = Group.objects.get(org=self.unicef, uuid="G-001", name="Customers", is_active=False) spammers = Group.objects.get(org=self.unicef, uuid="G-002", name="Spammers", is_active=False) self.assertEqual(bob.name, "Bob McFlow") self.assertEqual(bob.language, "eng") self.assertEqual(set(bob.groups.all()), {customers}) self.assertEqual(bob.get_fields(), {'age': "34"}) mock_get_contacts.side_effect = [ # first call to get active contacts will just one updated contact MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': "35"}, failed=True, blocked=False ) ] ), # second call to get deleted contacts returns Jim MockClientQuery( [ TembaContact.create( uuid="C-002", name=None, language=None, urns=[], groups=[], fields=None, failed=True, blocked=False ), ] ) ] with self.assertNumQueries(12): self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 1, 1, 0)) self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim}) self.assertEqual(jim.groups.count(), 0) # de-activated contacts are removed from groups bob.refresh_from_db() self.assertEqual(bob.name, "Bob McFlough") self.assertEqual(bob.language, "fre") self.assertEqual(set(bob.groups.all()), {spammers}) self.assertEqual(bob.get_fields(), {'age': "35"}) mock_get_contacts.side_effect = [ # first call to get active contacts will return a contact with only a change to URNs.. which we don't track MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow22"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': "35"}, failed=True, blocked=False ) ] ), MockClientQuery([]) ] with self.assertNumQueries(2): self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 0, 1)) self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim}) mock_get_contacts.side_effect = [ # first call to get active contacts will show one contact is now blocked MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={'age': 35}, failed=True, blocked=True ) ] ), MockClientQuery([]) ] with self.assertNumQueries(5): self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 1, 0)) # blocked = deleted self.assertEqual(set(Contact.objects.filter(is_active=True)), {ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), {bob, jim})
def test_local_kwargs(self): temba_contact = TembaContact.create(uuid='C-006', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='Musicians'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': None, 'state': None, 'lga': None, 'occupation': None, 'born': None, 'gender': None}, language='eng') self.assertIsNone(self.syncer.local_kwargs(self.nigeria, temba_contact)) temba_contact = TembaContact.create(uuid='C-006', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': None, 'state': None, 'lga': None, 'occupation': None, 'born': None, 'gender': None}, language='eng') self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact), {'org': self.nigeria, 'uuid': 'C-006', 'gender': '', 'born': 0, 'occupation': '', 'registered_on': None, 'state': '', 'district': ''}) temba_contact = TembaContact.create(uuid='C-007', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state': 'Kigali', 'lga': 'Oyo', 'occupation': 'Student', 'born': '1990', 'gender': 'Male'}, language='eng') self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact), {'org': self.nigeria, 'uuid': 'C-007', 'gender': 'M', 'born': 1990, 'occupation': 'Student', 'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'), 'state': '', 'district': ''}) temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos', 'lga': 'Oyo', 'occupation': 'Student', 'born': '1990', 'gender': 'Male'}, language='eng') self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact), {'org': self.nigeria, 'uuid': 'C-008', 'gender': 'M', 'born': 1990, 'occupation': 'Student', 'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'), 'state': 'R-LAGOS', 'district': 'R-OYO'}) temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos', 'lga': 'Oyo', 'occupation': 'Student', 'born': '-1', 'gender': 'Male'}, language='eng') self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact), {'org': self.nigeria, 'uuid': 'C-008', 'gender': 'M', 'born': 0, 'occupation': 'Student', 'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'), 'state': 'R-LAGOS', 'district': 'R-OYO'}) temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'], groups=[ObjectRef.create(uuid='G-001', name='ureporters'), ObjectRef.create(uuid='G-007', name='Actors')], fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos', 'lga': 'Oyo', 'occupation': 'Student', 'born': '2147483648', 'gender': 'Male'}, language='eng') self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact), {'org': self.nigeria, 'uuid': 'C-008', 'gender': 'M', 'born': 0, 'occupation': 'Student', 'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'), 'state': 'R-LAGOS', 'district': 'R-OYO'})
def test_pull_messages(self, mock_get_messages): d1 = now() - timedelta(hours=10) d2 = now() - timedelta(hours=9) d3 = now() - timedelta(hours=8) d4 = now() - timedelta(hours=7) d5 = now() - timedelta(hours=6) mock_get_messages.side_effect = [ MockClientQuery( [ TembaMessage.create( id=101, contact=ObjectRef.create(uuid="C-001", name="Ann"), type="inbox", text="What is aids?", visibility="visible", labels=[ ObjectRef.create(uuid="L-001", name="AIDS"), # existing label ObjectRef.create(uuid="L-009", name="Flagged"), # pseudo-label ], created_on=d1, ), TembaMessage.create( id=102, contact=ObjectRef.create(uuid="C-002", name="Bob"), type="inbox", text="Can I catch Hiv?", visibility="visible", labels=[ObjectRef.create(uuid="L-007", name="Important")], # new label created_on=d2, ), TembaMessage.create( id=103, contact=ObjectRef.create(uuid="C-003", name="Cat"), type="inbox", text="I think I'm pregnant", visibility="visible", labels=[], created_on=d3, ), TembaMessage.create( id=104, contact=ObjectRef.create(uuid="C-004", name="Don"), type="flow", text="Php is amaze", visibility="visible", labels=[], created_on=d4, ), TembaMessage.create( id=105, contact=ObjectRef.create(uuid="C-005", name="Eve"), type="flow", text="Thanks for the pregnancy/HIV info", visibility="visible", labels=[], created_on=d5, ), ] ) ] self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (5, 0, 0, 0)) self.assertEqual(Contact.objects.filter(is_stub=False).count(), 2) self.assertEqual(Contact.objects.filter(is_stub=True).count(), 3) self.assertEqual(Message.objects.filter(is_handled=False).count(), 5) msg1 = Message.objects.get(backend_id=101, type="I", text="What is aids?", is_archived=False, is_flagged=True) important = Label.objects.get(org=self.unicef, uuid="L-007", name="Important") self.assertEqual(set(msg1.labels.all()), {self.aids}) # a message is updated in RapidPro mock_get_messages.side_effect = [ MockClientQuery( [ TembaMessage.create( id=101, contact=ObjectRef.create(uuid="C-001", name="Ann"), type="inbox", text="What is aids?", visibility="archived", labels=[ ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-007", name="Important"), ], created_on=d1, ) ] ) ] self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (0, 1, 0, 0)) msg1 = Message.objects.get(backend_id=101, type="I", text="What is aids?", is_archived=True, is_flagged=False) self.assertEqual(set(msg1.labels.all()), {self.aids, important})
def test_update_required(self): # create stub contact local = Contact.get_or_create(self.unicef, "C-001", "Ann") # a stub contact should always be updated self.assertTrue( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Ann", urns=["tel:1234"], groups=[], fields={}, language=None, blocked=False, stopped=False, modified_on=now(), ), {}, ) ) local.groups.add(self.reporters) local.language = "eng" local.is_stub = False local.fields = {"chat_name": "ann"} local.save() # no differences (besides null field value which is ignored) self.assertFalse( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Ann", urns=["tel:1234"], groups=[ObjectRef.create(uuid="G-003", name="Reporters")], fields={"chat_name": "ann", "age": None}, language="eng", blocked=False, stopped=False, modified_on=now(), ), {}, ) ) # name change self.assertTrue( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Annie", urns=["tel:1234"], groups=[ObjectRef.create(uuid="G-003", name="Reporters")], fields={"chat_name": "ann"}, language="eng", blocked=False, stopped=False, modified_on=now(), ), {}, ) ) # group change self.assertTrue( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Ann", urns=["tel:1234"], groups=[ObjectRef.create(uuid="G-002", name="Females")], fields={"chat_name": "ann"}, language="eng", blocked=False, stopped=False, modified_on=now(), ), {}, ) ) # field value change self.assertTrue( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Ann", urns=["tel:1234"], groups=[ObjectRef.create(uuid="G-003", name="Reporters")], fields={"chat_name": "ann8111"}, language="eng", blocked=False, stopped=False, modified_on=now(), ), {}, ) ) # new field self.assertTrue( self.syncer.update_required( local, TembaContact.create( uuid="000-001", name="Ann", urns=["tel:1234"], groups=[ObjectRef.create(uuid="G-003", name="Reporters")], fields={"chat_name": "ann", "age": "35"}, language="eng", blocked=False, stopped=False, modified_on=now(), ), {}, ) )
def test_pull_messages(self, mock_get_messages): d1 = now() - timedelta(hours=10) d2 = now() - timedelta(hours=9) d3 = now() - timedelta(hours=8) d4 = now() - timedelta(hours=7) d5 = now() - timedelta(hours=6) mock_get_messages.side_effect = [ MockClientQuery([ TembaMessage.create(id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), type='inbox', text="What is aids?", visibility="visible", labels=[ ObjectRef.create(uuid='L-001', name="AIDS"), # existing label ObjectRef.create(uuid='L-009', name="Flagged"), # pseudo-label ], created_on=d1), TembaMessage.create(id=102, contact=ObjectRef.create(uuid='C-002', name="Bob"), type='inbox', text="Can I catch Hiv?", visibility="visible", labels=[ ObjectRef.create(uuid='L-007', name="Important"), # new label ], created_on=d2), TembaMessage.create(id=103, contact=ObjectRef.create(uuid='C-003', name="Cat"), type='inbox', text="I think I'm pregnant", visibility="visible", labels=[], created_on=d3), TembaMessage.create(id=104, contact=ObjectRef.create(uuid='C-004', name="Don"), type='flow', text="Php is amaze", visibility="visible", labels=[], created_on=d4), TembaMessage.create(id=105, contact=ObjectRef.create(uuid='C-005', name="Eve"), type='flow', text="Thanks for the pregnancy/HIV info", visibility="visible", labels=[], created_on=d5) ]) ] self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (5, 0, 0, 0)) self.assertEqual(Contact.objects.filter(is_stub=False).count(), 2) self.assertEqual(Contact.objects.filter(is_stub=True).count(), 3) self.assertEqual(Message.objects.filter(is_handled=False).count(), 5) msg1 = Message.objects.get(backend_id=101, type='I', text="What is aids?", is_archived=False, is_flagged=True) important = Label.objects.get(org=self.unicef, uuid='L-007', name="Important") self.assertEqual(set(msg1.labels.all()), {self.aids}) # a message is updated in RapidPro mock_get_messages.side_effect = [ MockClientQuery([ TembaMessage.create(id=101, contact=ObjectRef.create(uuid='C-001', name="Ann"), type='inbox', text="What is aids?", visibility="archived", labels=[ ObjectRef.create(uuid='L-001', name="AIDS"), ObjectRef.create(uuid='L-007', name="Important") ], created_on=d1) ]) ] self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (0, 1, 0, 0)) msg1 = Message.objects.get(backend_id=101, type='I', text="What is aids?", is_archived=True, is_flagged=False) self.assertEqual(set(msg1.labels.all()), {self.aids, important})
def test_pull_contacts(self, mock_get_contacts): # start with nothing... Group.objects.all().delete() Field.objects.all().delete() Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={"age": "34"}, stopped=False, blocked=False, ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={"age": "67"}, stopped=False, blocked=False, ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={"age": "35"}, stopped=True, blocked=False, ) ], ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(16): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) bob = Contact.objects.get(uuid="C-001") jim = Contact.objects.get(uuid="C-002") ann = Contact.objects.get(uuid="C-003") self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, jim, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), set()) # stub contact groups will have been created too customers = Group.objects.get(org=self.unicef, uuid="G-001", name="Customers", is_active=False) spammers = Group.objects.get(org=self.unicef, uuid="G-002", name="Spammers", is_active=False) self.assertEqual(bob.name, "Bob McFlow") self.assertEqual(bob.language, "eng") self.assertEqual(set(bob.groups.all()), {customers}) self.assertEqual(bob.get_fields(), {"age": "34"}) mock_get_contacts.side_effect = [ # first call to get active contacts will just one updated contact MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={"age": "35"}, stopped=True, blocked=False, ) ] ), # second call to get deleted contacts returns Jim MockClientQuery( [ TembaContact.create( uuid="C-002", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(13): self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 1, 1, 0)) self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim}) self.assertEqual(jim.groups.count(), 0) # de-activated contacts are removed from groups bob.refresh_from_db() self.assertEqual(bob.name, "Bob McFlough") self.assertEqual(bob.language, "fre") self.assertEqual(set(bob.groups.all()), {spammers}) self.assertEqual(bob.get_fields(), {"age": "35"}) mock_get_contacts.side_effect = [ # first call to get active contacts will return a contact with only a change to URNs.. which we don't track MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow22"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={"age": "35"}, stopped=True, blocked=False, ) ] ), MockClientQuery([]), ] with self.assertNumQueries(3): self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 0, 1)) self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann}) self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim})
def test_update_required(self): # create stub contact local = Contact.get_or_create(self.unicef, 'C-001', "Ann") # a stub contact should always be updated self.assertTrue(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Ann", urns=['tel:1234'], groups=[], fields={}, language=None, modified_on=now() ), {})) local.groups.add(self.reporters) local.language = "eng" local.is_stub = False local.fields = {'chat_name': "ann"} local.save() # no differences (besides null field value which is ignored) self.assertFalse(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Ann", urns=['tel:1234'], groups=[ObjectRef.create(uuid='G-003', name="Reporters")], fields={'chat_name': "ann", 'age': None}, language='eng', modified_on=now() ), {})) # name change self.assertTrue(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Annie", urns=['tel:1234'], groups=[ObjectRef.create(uuid='G-003', name="Reporters")], fields={'chat_name': "ann"}, language='eng', modified_on=now() ), {})) # group change self.assertTrue(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Ann", urns=['tel:1234'], groups=[ObjectRef.create(uuid='G-002', name="Females")], fields={'chat_name': "ann"}, language='eng', modified_on=now() ), {})) # field value change self.assertTrue(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Ann", urns=['tel:1234'], groups=[ObjectRef.create(uuid='G-003', name="Reporters")], fields={'chat_name': "ann8111"}, language='eng', modified_on=now() ), {})) # new field self.assertTrue(self.syncer.update_required(local, TembaContact.create( uuid='000-001', name="Ann", urns=['tel:1234'], groups=[ObjectRef.create(uuid='G-003', name="Reporters")], fields={'chat_name': "ann", 'age': "35"}, language='eng', modified_on=now() ), {}))
def test_contact_sync(self, mock_get_groups, mock_get_fields, mock_get_contacts): # start with no groups or fields Group.objects.all().delete() Field.objects.all().delete() fetch_size = 250 num_fetches = 4 num_groups = 50 num_fields = 50 names = ["Ann", "Bob", "Cat"] field_values = ["12345", None] groups_in = 5 # setup get_fields fields = [ TembaField.create(key="field_%d" % f, label="Field #%d" % f, value_type="text") for f in range(0, num_fields) ] mock_get_fields.return_value = MockClientQuery(fields) # sync fields start = time.time() self.assertEqual((num_fields, 0, 0, 0), self.backend.pull_fields(self.unicef)) print("Initial field sync: %f secs" % (time.time() - start)) # setup get_groups groups = [ TembaGroup.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g, count=0) for g in range(0, num_groups) ] mock_get_groups.return_value = MockClientQuery(groups) # sync groups start = time.time() self.assertEqual((num_groups, 0, 0, 0), self.backend.pull_groups(self.unicef)) print("Initial group sync: %f secs" % (time.time() - start)) # setup get_contacts to return multiple fetches of contacts active_fetches = [] for b in range(0, num_fetches): batch = [] for c in range(0, fetch_size): num = b * fetch_size + c batch.append( TembaContact.create( uuid="C0000000-0000-0000-0000-00000000%04d" % num, name=names[num % len(names)], language="eng", urns=["tel:+26096415%04d" % num], groups=[ ObjectRef.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g) for g in range(0, groups_in) ], fields={ "custom_field_%d" % f: field_values[f % len(field_values)] for f in range(0, num_fields) }, stopped=False, blocked=False, ) ) active_fetches.append(batch) mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] # no deleted contacts start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) print("Initial contact sync: %f secs" % (time.time() - start)) self.assertEqual((num_created, num_updated, num_deleted), (num_fetches * fetch_size, 0, 0)) # slowest_queries = sorted(connection.queries, key=lambda q: q['time'], reverse=True)[:10] # for q in slowest_queries: # print "%s -- %s" % (q['time'], q['sql']) # simulate a subsequent sync with no changes mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, 0, 0)) print("Contact sync with no changes: %f secs" % (time.time() - start)) # simulate an update of 1 field value for batch in active_fetches: for c in batch: c.fields["custom_field_1"] = "UPDATED" mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0)) print("Contact sync with 1 field value changes: %f secs" % (time.time() - start)) # simulate an update of 10 field values for batch in active_fetches: for c in batch: for f in (10, 11, 12, 13, 14, 15, 16, 17, 18, 19): c.fields["custom_field_%d" % f] = "UPDATED" mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0)) print("Contact sync with 10 field value changes: %f secs" % (time.time() - start))
def test_local_kwargs(self): temba_contact = TembaContact.create( uuid="C-006", name="Jan", urns=["tel:123"], groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")], fields={ "registration_date": None, "state": None, "lga": None, "occupation": None, "born": None, "gender": None, }, language="eng", ) self.assertEqual( self.syncer.local_kwargs(self.nigeria, temba_contact), { "backend": self.floip_backend, "org": self.nigeria, "uuid": "C-006", "gender": "", "born": 0, "occupation": "", "registered_on": None, "state": "", "district": "", "ward": "", }, ) temba_contact = TembaContact.create( uuid="C-007", name="Jan", urns=["tel:123"], groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")], fields={ "registration_date": "2014-01-02T03:04:05.000000Z", "state": "Kigali", "lga": "Oyo", "occupation": "Student", "born": "1990", "gender": "Male", }, language="eng", ) self.assertEqual( self.syncer.local_kwargs(self.nigeria, temba_contact), { "backend": self.floip_backend, "org": self.nigeria, "uuid": "C-007", "gender": "M", "born": 1990, "occupation": "Student", "registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"), "state": "", "district": "", "ward": "", }, ) temba_contact = TembaContact.create( uuid="C-008", name="Jan", urns=["tel:123"], groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")], fields={ "registration_date": "2014-01-02T03:04:05.000000Z", "state": "Lagos", "lga": "Oyo", "ward": "Ikeja", "occupation": "Student", "born": "1990", "gender": "Male", }, language="eng", ) self.assertEqual( self.syncer.local_kwargs(self.nigeria, temba_contact), { "backend": self.floip_backend, "org": self.nigeria, "uuid": "C-008", "gender": "M", "born": 1990, "occupation": "Student", "registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"), "state": "R-LAGOS", "district": "R-OYO", "ward": "R-IKEJA", }, ) temba_contact = TembaContact.create( uuid="C-008", name="Jan", urns=["tel:123"], groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")], fields={ "registration_date": "2014-01-02T03:04:05.000000Z", "state": "Lagos", "lga": "Oyo", "occupation": "Student", "born": "-1", "gender": "Male", }, language="eng", ) self.assertEqual( self.syncer.local_kwargs(self.nigeria, temba_contact), { "backend": self.floip_backend, "org": self.nigeria, "uuid": "C-008", "gender": "M", "born": 0, "occupation": "Student", "registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"), "state": "R-LAGOS", "district": "R-OYO", "ward": "", }, ) temba_contact = TembaContact.create( uuid="C-008", name="Jan", urns=["tel:123"], groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")], fields={ "registration_date": "2014-01-02T03:04:05.000000Z", "state": "Lagos", "lga": "Oyo", "occupation": "Student", "born": "2147483648", "gender": "Male", }, language="eng", ) self.assertEqual( self.syncer.local_kwargs(self.nigeria, temba_contact), { "backend": self.floip_backend, "org": self.nigeria, "uuid": "C-008", "gender": "M", "born": 0, "occupation": "Student", "registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"), "state": "R-LAGOS", "district": "R-OYO", "ward": "", }, )
def test_pull_contacts(self, mock_get_contacts): Contact.objects.all().delete() # empty fetches mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery([]), # second call to get deleted contacts MockClientQuery([]), ] with self.assertNumQueries(0): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 0)) # fecthed contact not in configured group get ignored mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="Customers")], fields={"age": "34"}, stopped=False, blocked=False, ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={"age": "67"}, stopped=False, blocked=False, ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={"age": "35"}, stopped=True, blocked=False, ) ], ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "34"}, stopped=False, blocked=False, ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-002", name="Spammers")], fields={"age": "67"}, stopped=False, blocked=False, ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={"age": "35"}, stopped=True, blocked=False, ) ], ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 3)) Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "34"}, stopped=False, blocked=False, ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "67"}, stopped=False, blocked=False, ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[], fields={"age": "35"}, stopped=True, blocked=False, ) ], ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) Contact.objects.all().delete() mock_get_contacts.side_effect = [ # first call to get active contacts will return two fetches of 2 and 1 contacts # all included in the reporters MockClientQuery( [ TembaContact.create( uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "34"}, stopped=False, blocked=False, ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "67"}, stopped=False, blocked=False, ), ], [ TembaContact.create( uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "35"}, stopped=True, blocked=False, ) ], ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0)) contact_jan = Contact.objects.filter(uuid="C-001").first() self.assertFalse(contact_jan.born) self.assertFalse(contact_jan.state) mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery( [ TembaContact.create( uuid="C-001", name="Jan", urns=["tel:123"], groups=[ ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors"), ], fields={ "registration_date": "2014-01-02T03:04:05.000000Z", "state": "Nigeria > Lagos", "lga": "Nigeria > Lagos > Oyo", "occupation": "Student", "born": "1990", "gender": "Male", }, language="eng", ), TembaContact.create( uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"], groups=[ObjectRef.create(uuid="G-001", name="ureporters")], fields={"age": "67", "born": "1992"}, stopped=False, blocked=False, ), ] ), # second call to get deleted contacts returns a contact we don't have MockClientQuery( [ TembaContact.create( uuid="C-004", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(10): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 2, 0, 0)) contact_jan = Contact.objects.filter(uuid="C-001").first() self.assertTrue(contact_jan.born) self.assertEqual(contact_jan.born, 1990) self.assertTrue(contact_jan.state) self.assertEqual(contact_jan.state, "R-LAGOS") self.assertTrue(Contact.objects.filter(uuid="C-002", is_active=True)) mock_get_contacts.side_effect = [ # first call to get active contacts MockClientQuery([]), # second call to get deleted contacts MockClientQuery( [ TembaContact.create( uuid="C-002", name=None, language=None, urns=[], groups=[], fields=None, stopped=True, blocked=False, ) ] ), ] with self.assertNumQueries(3): num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None) self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 1, 0)) self.assertFalse(Contact.objects.filter(uuid="C-002", is_active=True))
def test_contact_sync(self, mock_get_groups, mock_get_fields, mock_get_contacts): # start with no groups or fields Group.objects.all().delete() Field.objects.all().delete() fetch_size = 250 num_fetches = 4 num_groups = 50 num_fields = 50 names = ["Ann", "Bob", "Cat"] field_values = ["12345", None] groups_in = 5 # setup get_fields fields = [TembaField.create(key='field_%d' % f, label='Field #%d' % f, value_type='text') for f in range(0, num_fields)] mock_get_fields.return_value = MockClientQuery(fields) # sync fields start = time.time() self.assertEqual((num_fields, 0, 0, 0), self.backend.pull_fields(self.unicef)) print "Initial field sync: %f secs" % (time.time() - start) # setup get_groups groups = [TembaGroup.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g, count=0) for g in range(0, num_groups)] mock_get_groups.return_value = MockClientQuery(groups) # sync groups start = time.time() self.assertEqual((num_groups, 0, 0, 0), self.backend.pull_groups(self.unicef)) print "Initial group sync: %f secs" % (time.time() - start) # setup get_contacts to return multiple fetches of contacts active_fetches = [] for b in range(0, num_fetches): batch = [] for c in range(0, fetch_size): num = b * fetch_size + c batch.append(TembaContact.create( uuid="C0000000-0000-0000-0000-00000000%04d" % num, name=names[num % len(names)], language="eng", urns=["tel:+26096415%04d" % num], groups=[ ObjectRef.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g) for g in range(0, groups_in) ], fields={'custom_field_%d' % f: field_values[f % len(field_values)] for f in range(0, num_fields)}, failed=False, blocked=False )) active_fetches.append(batch) mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] # no deleted contacts start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) print "Initial contact sync: %f secs" % (time.time() - start) self.assertEqual((num_created, num_updated, num_deleted), (num_fetches * fetch_size, 0, 0)) # slowest_queries = sorted(connection.queries, key=lambda q: q['time'], reverse=True)[:10] # for q in slowest_queries: # print "%s -- %s" % (q['time'], q['sql']) # simulate a subsequent sync with no changes mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, 0, 0)) print "Contact sync with no changes: %f secs" % (time.time() - start) # simulate an update of 1 field value for batch in active_fetches: for c in batch: c.fields['custom_field_1'] = "UPDATED" mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0)) print "Contact sync with 1 field value changes: %f secs" % (time.time() - start) # simulate an update of 10 field values for batch in active_fetches: for c in batch: for f in (10, 11, 12, 13, 14, 15, 16, 17, 18, 19): c.fields['custom_field_%d' % f] = "UPDATED" mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] start = time.time() num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None) self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0)) print "Contact sync with 10 field value changes: %f secs" % (time.time() - start)