def test_fetch_contact_messages(self, mock_get_messages):
d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC)
d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC)
d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC)
mock_get_messages.return_value = MockClientQuery([
TembaMessage.create(
id=101,
broadcast=201,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
text="What is AIDS?",
type='inbox',
direction='in',
visibility='visible',
labels=[ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-111", name="Flagged")],
created_on=d2
)
])
messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3)
self.assertEqual(messages, [{
'id': 101,
'contact': {'uuid': "C-001", 'name': "Ann"},
'text': "What is AIDS?",
'time': d2,
'labels': [{'id': self.aids.pk, 'name': "AIDS"}],
'flagged': True,
'archived': False,
'direction': 'I',
'flow': False,
'sender': None,
'broadcast': 201
}])
def test_update_required(self):
d1 = now() - timedelta(hours=1)
local = self.create_message(self.unicef, 101, self.ann, "Yes", [self.aids], is_flagged=False)
# remote message has been flagged
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS"), ObjectRef.create(uuid='L-009', name="Flagged")],
created_on=d1
), {}))
# remote message has been archived
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='archived',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS")],
created_on=d1
), {}))
# remote message has been relabelled
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='archived',
text="Yes",
labels=[ObjectRef.create(uuid='L-002', name="Pregnancy")],
created_on=d1
), {}))
# no differences
self.assertFalse(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS")],
created_on=d1
), {}))
def test_update_required(self):
d1 = now() - timedelta(hours=1)
local = self.create_message(self.unicef, 101, self.ann, "Yes", [self.aids], is_flagged=False)
# remote message has been flagged
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS"), ObjectRef.create(uuid='L-009', name="Flagged")],
created_on=d1
), {}))
# remote message has been archived
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='archived',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS")],
created_on=d1
), {}))
# remote message has been relabelled
self.assertTrue(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='archived',
text="Yes",
labels=[ObjectRef.create(uuid='L-002', name="Pregnancy")],
created_on=d1
), {}))
# no differences
self.assertFalse(self.syncer.update_required(local, TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="Yes",
labels=[ObjectRef.create(uuid='L-001', name="AIDS")],
created_on=d1
), {}))
def test_local_kwargs(self):
d1 = now() - timedelta(hours=1)
d2 = now() - timedelta(days=32)
remote = TembaMessage.create(
id=123456789,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
urn="twitter:ann123",
direction="in",
type="inbox",
status="handled",
visibility="visible",
text="I have lots of questions!",
labels=[ObjectRef.create(uuid="L-001", name="Spam"), ObjectRef.create(uuid="L-009", name="Flagged")],
created_on=d1,
)
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(
kwargs,
{
"org": self.unicef,
"backend_id": 123456789,
"type": "I",
"text": "I have lots of questions!",
"is_flagged": True,
"is_archived": False,
"created_on": d1,
"__data__contact": ("C-001", "Ann"),
"__data__labels": [("L-001", "Spam")],
},
)
# if remote is archived, so will local
remote.visibility = "archived"
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(kwargs["is_archived"], True)
# if syncer is set to save as handled, local will be marked as handled
kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs["is_handled"], True)
# if remote is too old, local will be marked as handled
remote.created_on = d2
kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs["is_handled"], True)
# if remote is deleted, should return none
remote.visibility = "deleted"
self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_local_kwargs(self):
d1 = now() - timedelta(hours=1)
d2 = now() - timedelta(days=32)
remote = TembaMessage.create(
id=123456789,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="I have lots of questions!",
labels=[ObjectRef.create(uuid='L-001', name="Spam"), ObjectRef.create(uuid='L-009', name="Flagged")],
created_on=d1
)
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(kwargs, {
'org': self.unicef,
'backend_id': 123456789,
'type': 'I',
'text': "I have lots of questions!",
'is_flagged': True,
'is_archived': False,
'created_on': d1,
'__data__contact': ("C-001", "Ann"),
'__data__labels': [("L-001", "Spam")],
})
# if remote is archived, so will local
remote.visibility = 'archived'
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(kwargs['is_archived'], True)
# if syncer is set to save as handled, local will be marked as handled
kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs['is_handled'], True)
# if remote is too old, local will be marked as handled
remote.created_on = d2
kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs['is_handled'], True)
# if remote is deleted, should return none
remote.visibility = 'deleted'
self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_local_kwargs(self):
d1 = now() - timedelta(hours=1)
d2 = now() - timedelta(days=32)
remote = TembaMessage.create(
id=123456789,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
urn="twitter:ann123",
direction='in',
type='inbox',
status='handled',
visibility='visible',
text="I have lots of questions!",
labels=[ObjectRef.create(uuid='L-001', name="Spam"), ObjectRef.create(uuid='L-009', name="Flagged")],
created_on=d1
)
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(kwargs, {
'org': self.unicef,
'backend_id': 123456789,
'type': 'I',
'text': "I have lots of questions!",
'is_flagged': True,
'is_archived': False,
'created_on': d1,
'__data__contact': ("C-001", "Ann"),
'__data__labels': [("L-001", "Spam")],
})
# if remote is archived, so will local
remote.visibility = 'archived'
kwargs = MessageSyncer().local_kwargs(self.unicef, remote)
self.assertEqual(kwargs['is_archived'], True)
# if syncer is set to save as handled, local will be marked as handled
kwargs = MessageSyncer(as_handled=True).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs['is_handled'], True)
# if remote is too old, local will be marked as handled
remote.created_on = d2
kwargs = MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote)
self.assertTrue(kwargs['is_handled'], True)
# if remote is deleted, should return none
remote.visibility = 'deleted'
self.assertIsNone(MessageSyncer(as_handled=False).local_kwargs(self.unicef, remote))
def test_fetch_contact_messages(self, mock_get_messages):
d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC)
d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC)
d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC)
mock_get_messages.return_value = MockClientQuery([
TembaMessage.create(
id=102,
broadcast=201,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
text="Welcome",
type='inbox',
direction='out',
visibility='visible',
labels=[],
created_on=d3
),
TembaMessage.create(
id=101,
broadcast=None,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
text="Hello",
type='inbox',
direction='in',
visibility='archived',
labels=[ObjectRef.create(uuid="L-001", name="AIDS"), ObjectRef.create(uuid="L-111", name="Flagged")],
created_on=d2
)
])
messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3)
self.assertEqual(messages, [
{
'id': 201, # id is the broadcast id
'contact': {'id': self.ann.pk, 'name': "Ann"},
'urn': None,
'text': "Welcome",
'time': d3,
'direction': 'O',
'case': None,
'sender': None,
}
])
# check that JSON schemas match local outgoing model
outgoing = self.create_outgoing(self.unicef, self.admin, 201, 'B', "Hello", self.ann)
self.assertEqual(messages[0].keys(), outgoing.as_json().keys())
def test_local_kwargs(self):
kwargs = self.syncer.local_kwargs(
self.unicef,
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
)
self.assertEqual(
kwargs,
{
"org": self.unicef,
"uuid": "C-001",
"name": "Bob McFlow",
"language": "eng",
"is_blocked": False,
"is_stopped": False,
"is_stub": False,
"fields": {"age": "34"},
"__data__groups": [("G-001", "Customers")],
},
)
def test_fetch_contact_messages(self, mock_get_messages):
d1 = datetime(2015, 1, 2, 13, 0, tzinfo=pytz.UTC)
d2 = datetime(2015, 1, 2, 14, 0, tzinfo=pytz.UTC)
d3 = datetime(2015, 1, 2, 15, 0, tzinfo=pytz.UTC)
mock_get_messages.return_value = MockClientQuery(
[
TembaMessage.create(
id=102,
broadcast=201,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
text="Welcome",
type="inbox",
direction="out",
visibility="visible",
labels=[],
created_on=d3,
),
TembaMessage.create(
id=101,
broadcast=None,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
text="Hello",
type="inbox",
direction="in",
visibility="archived",
labels=[
ObjectRef.create(uuid="L-001", name="AIDS"),
ObjectRef.create(uuid="L-111", name="Flagged"),
],
created_on=d2,
),
]
)
messages = self.backend.fetch_contact_messages(self.unicef, self.ann, d1, d3)
self.assertEqual(len(messages), 1)
self.assertIsInstance(messages[0], Outgoing)
self.assertEqual(messages[0].backend_broadcast_id, 201)
self.assertEqual(messages[0].contact, self.ann)
self.assertEqual(messages[0].text, "Welcome")
self.assertEqual(messages[0].created_on, d3)
def test_local_kwargs(self):
kwargs = self.syncer.local_kwargs(self.unicef, TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={'age': "34"},
failed=False,
blocked=False
))
self.assertEqual(kwargs, {
'org': self.unicef,
'uuid': "C-001",
'name': "Bob McFlow",
'language': "eng",
'is_stub': False,
'fields': {'age': "34"},
'__data__groups': [("G-001", "Customers")],
})
def test_local_kwargs(self):
kwargs = self.syncer.local_kwargs(self.unicef, TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={'age': "34"},
stopped=False,
blocked=False
))
self.assertEqual(kwargs, {
'org': self.unicef,
'uuid': "C-001",
'name': "Bob McFlow",
'language': "eng",
'is_blocked': False,
'is_stopped': False,
'is_stub': False,
'fields': {'age': "34"},
'__data__groups': [("G-001", "Customers")],
})
def test_pull_contacts(self, mock_get_contacts):
Contact.objects.all().delete()
# empty fetches
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery([]),
# second call to get deleted contacts
MockClientQuery([])
]
with self.assertNumQueries(0):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 0))
# fecthed contact not in configured group get ignored
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={'age': "34"}, failed=False, blocked=False
),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': "67"}, failed=False, blocked=False
),
],
[
TembaContact.create(
uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"],
groups=[],
fields={'age': "35"}, failed=True, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(4):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 3))
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "34"}, failed=False, blocked=False
),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': "67"}, failed=False, blocked=False
),
],
[
TembaContact.create(
uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"],
groups=[],
fields={'age': "35"}, failed=True, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(8):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (1, 0, 0, 2))
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "34"}, failed=False, blocked=False
),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "67"}, failed=False, blocked=False
),
],
[
TembaContact.create(
uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"],
groups=[],
fields={'age': "35"}, failed=True, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(9):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (2, 0, 0, 1))
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
# all included in the reporters
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "34"}, failed=False, blocked=False
),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "67"}, failed=False, blocked=False
),
],
[
TembaContact.create(
uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "35"}, failed=True, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
contact_jan = Contact.objects.filter(uuid='C-001').first()
self.assertFalse(contact_jan.born)
self.assertFalse(contact_jan.state)
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery(
[
TembaContact.create(uuid='C-001', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos',
'lga': 'Oyo', 'occupation': 'Student', 'born': '1990',
'gender': 'Male'},
language='eng'),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={'age': "67", "born": "1992"}, failed=False, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(8):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 2, 0, 0))
contact_jan = Contact.objects.filter(uuid='C-001').first()
self.assertTrue(contact_jan.born)
self.assertEqual(contact_jan.born, 1990)
self.assertTrue(contact_jan.state)
self.assertEqual(contact_jan.state, 'R-LAGOS')
self.assertTrue(Contact.objects.filter(uuid='C-002', is_active=True))
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery([]
),
# second call to get deleted contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-002", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(2):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 1, 0))
self.assertFalse(Contact.objects.filter(uuid='C-002', is_active=True))
def test_pull_contacts(self, mock_get_contacts):
# start with nothing...
Group.objects.all().delete()
Field.objects.all().delete()
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlow", language="eng", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={'age': "34"}, failed=False, blocked=False
),
TembaContact.create(
uuid="C-002", name="Jim McMsg", language="fre", urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': "67"}, failed=False, blocked=False
),
],
[
TembaContact.create(
uuid="C-003", name="Ann McPoll", language="eng", urns=["tel:+250783835664"],
groups=[],
fields={'age': "35"}, failed=True, blocked=False
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(15):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
bob = Contact.objects.get(uuid="C-001")
jim = Contact.objects.get(uuid="C-002")
ann = Contact.objects.get(uuid="C-003")
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, jim, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), set())
# stub contact groups will have been created too
customers = Group.objects.get(org=self.unicef, uuid="G-001", name="Customers", is_active=False)
spammers = Group.objects.get(org=self.unicef, uuid="G-002", name="Spammers", is_active=False)
self.assertEqual(bob.name, "Bob McFlow")
self.assertEqual(bob.language, "eng")
self.assertEqual(set(bob.groups.all()), {customers})
self.assertEqual(bob.get_fields(), {'age': "34"})
mock_get_contacts.side_effect = [
# first call to get active contacts will just one updated contact
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': "35"}, failed=True, blocked=False
)
]
),
# second call to get deleted contacts returns Jim
MockClientQuery(
[
TembaContact.create(
uuid="C-002", name=None, language=None, urns=[], groups=[],
fields=None, failed=True, blocked=False
),
]
)
]
with self.assertNumQueries(12):
self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 1, 1, 0))
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim})
self.assertEqual(jim.groups.count(), 0) # de-activated contacts are removed from groups
bob.refresh_from_db()
self.assertEqual(bob.name, "Bob McFlough")
self.assertEqual(bob.language, "fre")
self.assertEqual(set(bob.groups.all()), {spammers})
self.assertEqual(bob.get_fields(), {'age': "35"})
mock_get_contacts.side_effect = [
# first call to get active contacts will return a contact with only a change to URNs.. which we don't track
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow22"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': "35"}, failed=True, blocked=False
)
]
),
MockClientQuery([])
]
with self.assertNumQueries(2):
self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 0, 1))
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim})
mock_get_contacts.side_effect = [
# first call to get active contacts will show one contact is now blocked
MockClientQuery(
[
TembaContact.create(
uuid="C-001", name="Bob McFlough", language="fre", urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={'age': 35}, failed=True, blocked=True
)
]
),
MockClientQuery([])
]
with self.assertNumQueries(5):
self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 1, 0)) # blocked = deleted
self.assertEqual(set(Contact.objects.filter(is_active=True)), {ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), {bob, jim})
def test_local_kwargs(self):
temba_contact = TembaContact.create(uuid='C-006', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='Musicians'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': None, 'state': None,
'lga': None, 'occupation': None, 'born': None,
'gender': None},
language='eng')
self.assertIsNone(self.syncer.local_kwargs(self.nigeria, temba_contact))
temba_contact = TembaContact.create(uuid='C-006', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': None, 'state': None,
'lga': None, 'occupation': None, 'born': None,
'gender': None},
language='eng')
self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact),
{'org': self.nigeria,
'uuid': 'C-006',
'gender': '',
'born': 0,
'occupation': '',
'registered_on': None,
'state': '',
'district': ''})
temba_contact = TembaContact.create(uuid='C-007', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': '2014-01-02T03:04:05.000000Z',
'state': 'Kigali', 'lga': 'Oyo', 'occupation': 'Student',
'born': '1990', 'gender': 'Male'},
language='eng')
self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact),
{'org': self.nigeria,
'uuid': 'C-007',
'gender': 'M',
'born': 1990,
'occupation': 'Student',
'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'),
'state': '',
'district': ''})
temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos',
'lga': 'Oyo', 'occupation': 'Student', 'born': '1990',
'gender': 'Male'},
language='eng')
self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact),
{'org': self.nigeria,
'uuid': 'C-008',
'gender': 'M',
'born': 1990,
'occupation': 'Student',
'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'),
'state': 'R-LAGOS',
'district': 'R-OYO'})
temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos',
'lga': 'Oyo', 'occupation': 'Student', 'born': '-1',
'gender': 'Male'},
language='eng')
self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact),
{'org': self.nigeria,
'uuid': 'C-008',
'gender': 'M',
'born': 0,
'occupation': 'Student',
'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'),
'state': 'R-LAGOS',
'district': 'R-OYO'})
temba_contact = TembaContact.create(uuid='C-008', name="Jan", urns=['tel:123'],
groups=[ObjectRef.create(uuid='G-001', name='ureporters'),
ObjectRef.create(uuid='G-007', name='Actors')],
fields={'registration_date': '2014-01-02T03:04:05.000000Z', 'state':'Lagos',
'lga': 'Oyo', 'occupation': 'Student', 'born': '2147483648',
'gender': 'Male'},
language='eng')
self.assertEqual(self.syncer.local_kwargs(self.nigeria, temba_contact),
{'org': self.nigeria,
'uuid': 'C-008',
'gender': 'M',
'born': 0,
'occupation': 'Student',
'registered_on': json_date_to_datetime('2014-01-02T03:04:05.000'),
'state': 'R-LAGOS',
'district': 'R-OYO'})
def test_pull_messages(self, mock_get_messages):
d1 = now() - timedelta(hours=10)
d2 = now() - timedelta(hours=9)
d3 = now() - timedelta(hours=8)
d4 = now() - timedelta(hours=7)
d5 = now() - timedelta(hours=6)
mock_get_messages.side_effect = [
MockClientQuery(
[
TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
type="inbox",
text="What is aids?",
visibility="visible",
labels=[
ObjectRef.create(uuid="L-001", name="AIDS"), # existing label
ObjectRef.create(uuid="L-009", name="Flagged"), # pseudo-label
],
created_on=d1,
),
TembaMessage.create(
id=102,
contact=ObjectRef.create(uuid="C-002", name="Bob"),
type="inbox",
text="Can I catch Hiv?",
visibility="visible",
labels=[ObjectRef.create(uuid="L-007", name="Important")], # new label
created_on=d2,
),
TembaMessage.create(
id=103,
contact=ObjectRef.create(uuid="C-003", name="Cat"),
type="inbox",
text="I think I'm pregnant",
visibility="visible",
labels=[],
created_on=d3,
),
TembaMessage.create(
id=104,
contact=ObjectRef.create(uuid="C-004", name="Don"),
type="flow",
text="Php is amaze",
visibility="visible",
labels=[],
created_on=d4,
),
TembaMessage.create(
id=105,
contact=ObjectRef.create(uuid="C-005", name="Eve"),
type="flow",
text="Thanks for the pregnancy/HIV info",
visibility="visible",
labels=[],
created_on=d5,
),
]
)
]
self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (5, 0, 0, 0))
self.assertEqual(Contact.objects.filter(is_stub=False).count(), 2)
self.assertEqual(Contact.objects.filter(is_stub=True).count(), 3)
self.assertEqual(Message.objects.filter(is_handled=False).count(), 5)
msg1 = Message.objects.get(backend_id=101, type="I", text="What is aids?", is_archived=False, is_flagged=True)
important = Label.objects.get(org=self.unicef, uuid="L-007", name="Important")
self.assertEqual(set(msg1.labels.all()), {self.aids})
# a message is updated in RapidPro
mock_get_messages.side_effect = [
MockClientQuery(
[
TembaMessage.create(
id=101,
contact=ObjectRef.create(uuid="C-001", name="Ann"),
type="inbox",
text="What is aids?",
visibility="archived",
labels=[
ObjectRef.create(uuid="L-001", name="AIDS"),
ObjectRef.create(uuid="L-007", name="Important"),
],
created_on=d1,
)
]
)
]
self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (0, 1, 0, 0))
msg1 = Message.objects.get(backend_id=101, type="I", text="What is aids?", is_archived=True, is_flagged=False)
self.assertEqual(set(msg1.labels.all()), {self.aids, important})
def test_update_required(self):
# create stub contact
local = Contact.get_or_create(self.unicef, "C-001", "Ann")
# a stub contact should always be updated
self.assertTrue(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Ann",
urns=["tel:1234"],
groups=[],
fields={},
language=None,
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
local.groups.add(self.reporters)
local.language = "eng"
local.is_stub = False
local.fields = {"chat_name": "ann"}
local.save()
# no differences (besides null field value which is ignored)
self.assertFalse(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Ann",
urns=["tel:1234"],
groups=[ObjectRef.create(uuid="G-003", name="Reporters")],
fields={"chat_name": "ann", "age": None},
language="eng",
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
# name change
self.assertTrue(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Annie",
urns=["tel:1234"],
groups=[ObjectRef.create(uuid="G-003", name="Reporters")],
fields={"chat_name": "ann"},
language="eng",
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
# group change
self.assertTrue(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Ann",
urns=["tel:1234"],
groups=[ObjectRef.create(uuid="G-002", name="Females")],
fields={"chat_name": "ann"},
language="eng",
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
# field value change
self.assertTrue(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Ann",
urns=["tel:1234"],
groups=[ObjectRef.create(uuid="G-003", name="Reporters")],
fields={"chat_name": "ann8111"},
language="eng",
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
# new field
self.assertTrue(
self.syncer.update_required(
local,
TembaContact.create(
uuid="000-001",
name="Ann",
urns=["tel:1234"],
groups=[ObjectRef.create(uuid="G-003", name="Reporters")],
fields={"chat_name": "ann", "age": "35"},
language="eng",
blocked=False,
stopped=False,
modified_on=now(),
),
{},
)
)
def test_pull_messages(self, mock_get_messages):
d1 = now() - timedelta(hours=10)
d2 = now() - timedelta(hours=9)
d3 = now() - timedelta(hours=8)
d4 = now() - timedelta(hours=7)
d5 = now() - timedelta(hours=6)
mock_get_messages.side_effect = [
MockClientQuery([
TembaMessage.create(id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
type='inbox',
text="What is aids?",
visibility="visible",
labels=[
ObjectRef.create(uuid='L-001', name="AIDS"), # existing label
ObjectRef.create(uuid='L-009', name="Flagged"), # pseudo-label
],
created_on=d1),
TembaMessage.create(id=102,
contact=ObjectRef.create(uuid='C-002', name="Bob"),
type='inbox',
text="Can I catch Hiv?",
visibility="visible",
labels=[
ObjectRef.create(uuid='L-007', name="Important"), # new label
],
created_on=d2),
TembaMessage.create(id=103,
contact=ObjectRef.create(uuid='C-003', name="Cat"),
type='inbox',
text="I think I'm pregnant",
visibility="visible",
labels=[],
created_on=d3),
TembaMessage.create(id=104,
contact=ObjectRef.create(uuid='C-004', name="Don"),
type='flow',
text="Php is amaze",
visibility="visible",
labels=[],
created_on=d4),
TembaMessage.create(id=105,
contact=ObjectRef.create(uuid='C-005', name="Eve"),
type='flow',
text="Thanks for the pregnancy/HIV info",
visibility="visible",
labels=[],
created_on=d5)
])
]
self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (5, 0, 0, 0))
self.assertEqual(Contact.objects.filter(is_stub=False).count(), 2)
self.assertEqual(Contact.objects.filter(is_stub=True).count(), 3)
self.assertEqual(Message.objects.filter(is_handled=False).count(), 5)
msg1 = Message.objects.get(backend_id=101, type='I', text="What is aids?", is_archived=False, is_flagged=True)
important = Label.objects.get(org=self.unicef, uuid='L-007', name="Important")
self.assertEqual(set(msg1.labels.all()), {self.aids})
# a message is updated in RapidPro
mock_get_messages.side_effect = [
MockClientQuery([
TembaMessage.create(id=101,
contact=ObjectRef.create(uuid='C-001', name="Ann"),
type='inbox',
text="What is aids?",
visibility="archived",
labels=[
ObjectRef.create(uuid='L-001', name="AIDS"),
ObjectRef.create(uuid='L-007', name="Important")
],
created_on=d1)
])
]
self.assertEqual(self.backend.pull_messages(self.unicef, d1, d5), (0, 1, 0, 0))
msg1 = Message.objects.get(backend_id=101, type='I', text="What is aids?", is_archived=True, is_flagged=False)
self.assertEqual(set(msg1.labels.all()), {self.aids, important})
def test_pull_contacts(self, mock_get_contacts):
# start with nothing...
Group.objects.all().delete()
Field.objects.all().delete()
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={"age": "67"},
stopped=False,
blocked=False,
),
],
[
TembaContact.create(
uuid="C-003",
name="Ann McPoll",
language="eng",
urns=["tel:+250783835664"],
groups=[],
fields={"age": "35"},
stopped=True,
blocked=False,
)
],
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(16):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
bob = Contact.objects.get(uuid="C-001")
jim = Contact.objects.get(uuid="C-002")
ann = Contact.objects.get(uuid="C-003")
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, jim, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), set())
# stub contact groups will have been created too
customers = Group.objects.get(org=self.unicef, uuid="G-001", name="Customers", is_active=False)
spammers = Group.objects.get(org=self.unicef, uuid="G-002", name="Spammers", is_active=False)
self.assertEqual(bob.name, "Bob McFlow")
self.assertEqual(bob.language, "eng")
self.assertEqual(set(bob.groups.all()), {customers})
self.assertEqual(bob.get_fields(), {"age": "34"})
mock_get_contacts.side_effect = [
# first call to get active contacts will just one updated contact
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlough",
language="fre",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={"age": "35"},
stopped=True,
blocked=False,
)
]
),
# second call to get deleted contacts returns Jim
MockClientQuery(
[
TembaContact.create(
uuid="C-002",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(13):
self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 1, 1, 0))
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim})
self.assertEqual(jim.groups.count(), 0) # de-activated contacts are removed from groups
bob.refresh_from_db()
self.assertEqual(bob.name, "Bob McFlough")
self.assertEqual(bob.language, "fre")
self.assertEqual(set(bob.groups.all()), {spammers})
self.assertEqual(bob.get_fields(), {"age": "35"})
mock_get_contacts.side_effect = [
# first call to get active contacts will return a contact with only a change to URNs.. which we don't track
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlough",
language="fre",
urns=["twitter:bobflow22"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={"age": "35"},
stopped=True,
blocked=False,
)
]
),
MockClientQuery([]),
]
with self.assertNumQueries(3):
self.assertEqual(self.backend.pull_contacts(self.unicef, None, None), (0, 0, 0, 1))
self.assertEqual(set(Contact.objects.filter(is_active=True)), {bob, ann})
self.assertEqual(set(Contact.objects.filter(is_active=False)), {jim})
def test_update_required(self):
# create stub contact
local = Contact.get_or_create(self.unicef, 'C-001', "Ann")
# a stub contact should always be updated
self.assertTrue(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=[],
fields={},
language=None,
modified_on=now()
), {}))
local.groups.add(self.reporters)
local.language = "eng"
local.is_stub = False
local.fields = {'chat_name': "ann"}
local.save()
# no differences (besides null field value which is ignored)
self.assertFalse(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=[ObjectRef.create(uuid='G-003', name="Reporters")],
fields={'chat_name': "ann", 'age': None},
language='eng',
modified_on=now()
), {}))
# name change
self.assertTrue(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Annie",
urns=['tel:1234'],
groups=[ObjectRef.create(uuid='G-003', name="Reporters")],
fields={'chat_name': "ann"},
language='eng',
modified_on=now()
), {}))
# group change
self.assertTrue(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=[ObjectRef.create(uuid='G-002', name="Females")],
fields={'chat_name': "ann"},
language='eng', modified_on=now()
), {}))
# field value change
self.assertTrue(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=[ObjectRef.create(uuid='G-003', name="Reporters")],
fields={'chat_name': "ann8111"},
language='eng', modified_on=now()
), {}))
# new field
self.assertTrue(self.syncer.update_required(local, TembaContact.create(
uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=[ObjectRef.create(uuid='G-003', name="Reporters")],
fields={'chat_name': "ann", 'age': "35"},
language='eng', modified_on=now()
), {}))
def test_contact_sync(self, mock_get_groups, mock_get_fields, mock_get_contacts):
# start with no groups or fields
Group.objects.all().delete()
Field.objects.all().delete()
fetch_size = 250
num_fetches = 4
num_groups = 50
num_fields = 50
names = ["Ann", "Bob", "Cat"]
field_values = ["12345", None]
groups_in = 5
# setup get_fields
fields = [
TembaField.create(key="field_%d" % f, label="Field #%d" % f, value_type="text")
for f in range(0, num_fields)
]
mock_get_fields.return_value = MockClientQuery(fields)
# sync fields
start = time.time()
self.assertEqual((num_fields, 0, 0, 0), self.backend.pull_fields(self.unicef))
print("Initial field sync: %f secs" % (time.time() - start))
# setup get_groups
groups = [
TembaGroup.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g, count=0)
for g in range(0, num_groups)
]
mock_get_groups.return_value = MockClientQuery(groups)
# sync groups
start = time.time()
self.assertEqual((num_groups, 0, 0, 0), self.backend.pull_groups(self.unicef))
print("Initial group sync: %f secs" % (time.time() - start))
# setup get_contacts to return multiple fetches of contacts
active_fetches = []
for b in range(0, num_fetches):
batch = []
for c in range(0, fetch_size):
num = b * fetch_size + c
batch.append(
TembaContact.create(
uuid="C0000000-0000-0000-0000-00000000%04d" % num,
name=names[num % len(names)],
language="eng",
urns=["tel:+26096415%04d" % num],
groups=[
ObjectRef.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g)
for g in range(0, groups_in)
],
fields={
"custom_field_%d" % f: field_values[f % len(field_values)] for f in range(0, num_fields)
},
stopped=False,
blocked=False,
)
)
active_fetches.append(batch)
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] # no deleted contacts
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
print("Initial contact sync: %f secs" % (time.time() - start))
self.assertEqual((num_created, num_updated, num_deleted), (num_fetches * fetch_size, 0, 0))
# slowest_queries = sorted(connection.queries, key=lambda q: q['time'], reverse=True)[:10]
# for q in slowest_queries:
# print "%s -- %s" % (q['time'], q['sql'])
# simulate a subsequent sync with no changes
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, 0, 0))
print("Contact sync with no changes: %f secs" % (time.time() - start))
# simulate an update of 1 field value
for batch in active_fetches:
for c in batch:
c.fields["custom_field_1"] = "UPDATED"
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0))
print("Contact sync with 1 field value changes: %f secs" % (time.time() - start))
# simulate an update of 10 field values
for batch in active_fetches:
for c in batch:
for f in (10, 11, 12, 13, 14, 15, 16, 17, 18, 19):
c.fields["custom_field_%d" % f] = "UPDATED"
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0))
print("Contact sync with 10 field value changes: %f secs" % (time.time() - start))
def test_local_kwargs(self):
temba_contact = TembaContact.create(
uuid="C-006",
name="Jan",
urns=["tel:123"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")],
fields={
"registration_date": None,
"state": None,
"lga": None,
"occupation": None,
"born": None,
"gender": None,
},
language="eng",
)
self.assertEqual(
self.syncer.local_kwargs(self.nigeria, temba_contact),
{
"backend": self.floip_backend,
"org": self.nigeria,
"uuid": "C-006",
"gender": "",
"born": 0,
"occupation": "",
"registered_on": None,
"state": "",
"district": "",
"ward": "",
},
)
temba_contact = TembaContact.create(
uuid="C-007",
name="Jan",
urns=["tel:123"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")],
fields={
"registration_date": "2014-01-02T03:04:05.000000Z",
"state": "Kigali",
"lga": "Oyo",
"occupation": "Student",
"born": "1990",
"gender": "Male",
},
language="eng",
)
self.assertEqual(
self.syncer.local_kwargs(self.nigeria, temba_contact),
{
"backend": self.floip_backend,
"org": self.nigeria,
"uuid": "C-007",
"gender": "M",
"born": 1990,
"occupation": "Student",
"registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"),
"state": "",
"district": "",
"ward": "",
},
)
temba_contact = TembaContact.create(
uuid="C-008",
name="Jan",
urns=["tel:123"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")],
fields={
"registration_date": "2014-01-02T03:04:05.000000Z",
"state": "Lagos",
"lga": "Oyo",
"ward": "Ikeja",
"occupation": "Student",
"born": "1990",
"gender": "Male",
},
language="eng",
)
self.assertEqual(
self.syncer.local_kwargs(self.nigeria, temba_contact),
{
"backend": self.floip_backend,
"org": self.nigeria,
"uuid": "C-008",
"gender": "M",
"born": 1990,
"occupation": "Student",
"registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"),
"state": "R-LAGOS",
"district": "R-OYO",
"ward": "R-IKEJA",
},
)
temba_contact = TembaContact.create(
uuid="C-008",
name="Jan",
urns=["tel:123"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")],
fields={
"registration_date": "2014-01-02T03:04:05.000000Z",
"state": "Lagos",
"lga": "Oyo",
"occupation": "Student",
"born": "-1",
"gender": "Male",
},
language="eng",
)
self.assertEqual(
self.syncer.local_kwargs(self.nigeria, temba_contact),
{
"backend": self.floip_backend,
"org": self.nigeria,
"uuid": "C-008",
"gender": "M",
"born": 0,
"occupation": "Student",
"registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"),
"state": "R-LAGOS",
"district": "R-OYO",
"ward": "",
},
)
temba_contact = TembaContact.create(
uuid="C-008",
name="Jan",
urns=["tel:123"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters"), ObjectRef.create(uuid="G-007", name="Actors")],
fields={
"registration_date": "2014-01-02T03:04:05.000000Z",
"state": "Lagos",
"lga": "Oyo",
"occupation": "Student",
"born": "2147483648",
"gender": "Male",
},
language="eng",
)
self.assertEqual(
self.syncer.local_kwargs(self.nigeria, temba_contact),
{
"backend": self.floip_backend,
"org": self.nigeria,
"uuid": "C-008",
"gender": "M",
"born": 0,
"occupation": "Student",
"registered_on": json_date_to_datetime("2014-01-02T03:04:05.000"),
"state": "R-LAGOS",
"district": "R-OYO",
"ward": "",
},
)
def test_pull_contacts(self, mock_get_contacts):
Contact.objects.all().delete()
# empty fetches
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery([]),
# second call to get deleted contacts
MockClientQuery([]),
]
with self.assertNumQueries(0):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 0))
# fecthed contact not in configured group get ignored
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="Customers")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={"age": "67"},
stopped=False,
blocked=False,
),
],
[
TembaContact.create(
uuid="C-003",
name="Ann McPoll",
language="eng",
urns=["tel:+250783835664"],
groups=[],
fields={"age": "35"},
stopped=True,
blocked=False,
)
],
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-002", name="Spammers")],
fields={"age": "67"},
stopped=False,
blocked=False,
),
],
[
TembaContact.create(
uuid="C-003",
name="Ann McPoll",
language="eng",
urns=["tel:+250783835664"],
groups=[],
fields={"age": "35"},
stopped=True,
blocked=False,
)
],
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 0, 3))
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "67"},
stopped=False,
blocked=False,
),
],
[
TembaContact.create(
uuid="C-003",
name="Ann McPoll",
language="eng",
urns=["tel:+250783835664"],
groups=[],
fields={"age": "35"},
stopped=True,
blocked=False,
)
],
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
Contact.objects.all().delete()
mock_get_contacts.side_effect = [
# first call to get active contacts will return two fetches of 2 and 1 contacts
# all included in the reporters
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Bob McFlow",
language="eng",
urns=["twitter:bobflow"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "34"},
stopped=False,
blocked=False,
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "67"},
stopped=False,
blocked=False,
),
],
[
TembaContact.create(
uuid="C-003",
name="Ann McPoll",
language="eng",
urns=["tel:+250783835664"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "35"},
stopped=True,
blocked=False,
)
],
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (3, 0, 0, 0))
contact_jan = Contact.objects.filter(uuid="C-001").first()
self.assertFalse(contact_jan.born)
self.assertFalse(contact_jan.state)
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-001",
name="Jan",
urns=["tel:123"],
groups=[
ObjectRef.create(uuid="G-001", name="ureporters"),
ObjectRef.create(uuid="G-007", name="Actors"),
],
fields={
"registration_date": "2014-01-02T03:04:05.000000Z",
"state": "Nigeria > Lagos",
"lga": "Nigeria > Lagos > Oyo",
"occupation": "Student",
"born": "1990",
"gender": "Male",
},
language="eng",
),
TembaContact.create(
uuid="C-002",
name="Jim McMsg",
language="fre",
urns=["tel:+250783835665"],
groups=[ObjectRef.create(uuid="G-001", name="ureporters")],
fields={"age": "67", "born": "1992"},
stopped=False,
blocked=False,
),
]
),
# second call to get deleted contacts returns a contact we don't have
MockClientQuery(
[
TembaContact.create(
uuid="C-004",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(10):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 2, 0, 0))
contact_jan = Contact.objects.filter(uuid="C-001").first()
self.assertTrue(contact_jan.born)
self.assertEqual(contact_jan.born, 1990)
self.assertTrue(contact_jan.state)
self.assertEqual(contact_jan.state, "R-LAGOS")
self.assertTrue(Contact.objects.filter(uuid="C-002", is_active=True))
mock_get_contacts.side_effect = [
# first call to get active contacts
MockClientQuery([]),
# second call to get deleted contacts
MockClientQuery(
[
TembaContact.create(
uuid="C-002",
name=None,
language=None,
urns=[],
groups=[],
fields=None,
stopped=True,
blocked=False,
)
]
),
]
with self.assertNumQueries(3):
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.nigeria, None, None)
self.assertEqual((num_created, num_updated, num_deleted, num_ignored), (0, 0, 1, 0))
self.assertFalse(Contact.objects.filter(uuid="C-002", is_active=True))
def test_contact_sync(self, mock_get_groups, mock_get_fields, mock_get_contacts):
# start with no groups or fields
Group.objects.all().delete()
Field.objects.all().delete()
fetch_size = 250
num_fetches = 4
num_groups = 50
num_fields = 50
names = ["Ann", "Bob", "Cat"]
field_values = ["12345", None]
groups_in = 5
# setup get_fields
fields = [TembaField.create(key='field_%d' % f, label='Field #%d' % f, value_type='text')
for f in range(0, num_fields)]
mock_get_fields.return_value = MockClientQuery(fields)
# sync fields
start = time.time()
self.assertEqual((num_fields, 0, 0, 0), self.backend.pull_fields(self.unicef))
print "Initial field sync: %f secs" % (time.time() - start)
# setup get_groups
groups = [TembaGroup.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g, count=0)
for g in range(0, num_groups)]
mock_get_groups.return_value = MockClientQuery(groups)
# sync groups
start = time.time()
self.assertEqual((num_groups, 0, 0, 0), self.backend.pull_groups(self.unicef))
print "Initial group sync: %f secs" % (time.time() - start)
# setup get_contacts to return multiple fetches of contacts
active_fetches = []
for b in range(0, num_fetches):
batch = []
for c in range(0, fetch_size):
num = b * fetch_size + c
batch.append(TembaContact.create(
uuid="C0000000-0000-0000-0000-00000000%04d" % num,
name=names[num % len(names)],
language="eng",
urns=["tel:+26096415%04d" % num],
groups=[
ObjectRef.create(uuid="G0000000-0000-0000-0000-00000000%04d" % g, name="Group #%d" % g)
for g in range(0, groups_in)
],
fields={'custom_field_%d' % f: field_values[f % len(field_values)] for f in range(0, num_fields)},
failed=False,
blocked=False
))
active_fetches.append(batch)
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])] # no deleted contacts
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
print "Initial contact sync: %f secs" % (time.time() - start)
self.assertEqual((num_created, num_updated, num_deleted), (num_fetches * fetch_size, 0, 0))
# slowest_queries = sorted(connection.queries, key=lambda q: q['time'], reverse=True)[:10]
# for q in slowest_queries:
# print "%s -- %s" % (q['time'], q['sql'])
# simulate a subsequent sync with no changes
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, 0, 0))
print "Contact sync with no changes: %f secs" % (time.time() - start)
# simulate an update of 1 field value
for batch in active_fetches:
for c in batch:
c.fields['custom_field_1'] = "UPDATED"
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0))
print "Contact sync with 1 field value changes: %f secs" % (time.time() - start)
# simulate an update of 10 field values
for batch in active_fetches:
for c in batch:
for f in (10, 11, 12, 13, 14, 15, 16, 17, 18, 19):
c.fields['custom_field_%d' % f] = "UPDATED"
mock_get_contacts.side_effect = [MockClientQuery(*active_fetches), MockClientQuery([])]
start = time.time()
num_created, num_updated, num_deleted, num_ignored = self.backend.pull_contacts(self.unicef, None, None)
self.assertEqual((num_created, num_updated, num_deleted), (0, num_fetches * fetch_size, 0))
print "Contact sync with 10 field value changes: %f secs" % (time.time() - start)