def fetch_contacts_task(org_id=None, fetch_all=False):
r = get_redis_connection()
key = 'fetch_contacts'
lock_timeout = 3600
if org_id:
key = 'fetch_contacts:%d' % org_id
lock_timeout = 300
if not r.get(key):
with r.lock(key, timeout=lock_timeout):
active_orgs = Org.objects.filter(is_active=True)
if org_id:
active_orgs = Org.objects.filter(pk=org_id)
for org in active_orgs:
start = time.time()
last_fetched_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % org.id
after = cache.get(last_fetched_key, None)
if after:
after = json_date_to_datetime(after)
if fetch_all:
after = None
try:
if after is None:
Boundary.fetch_boundaries(org)
ContactField.fetch_contact_fields(org)
Boundary.get_boundaries(org)
ContactField.get_contact_fields(org)
Contact.fetch_contacts(org, after=after)
print "Task: fetch_contacts for %s took %ss" % (org.name, time.time() - start)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception("Error fetching contacts: %s" % str(e))
def test_fetch_contacts(self):
self.nigeria.set_config('reporter_group', 'Reporters')
tz = pytz.timezone('UTC')
with patch.object(timezone, 'now', return_value=tz.localize(datetime(2015, 9, 29, 10, 20, 30, 40))):
with patch('dash.orgs.models.TembaClient1.get_groups') as mock_groups:
group = TembaGroup.create(uuid="uuid-8", name='reporters', size=120)
mock_groups.return_value = [group]
with patch('dash.orgs.models.TembaClient1.get_contacts') as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001', name="Ann", urns=['tel:1234'], groups=['000-002'],
fields=dict(state="Lagos", lga="Oyo", gender='Female', born="1990"),
language='eng',
modified_on=datetime(2015, 9, 20, 10, 20, 30, 400000, pytz.utc))]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertEqual(seen_uuids, [])
group = TembaGroup.create(uuid="000-002", name='reporters', size=120)
mock_groups.return_value = [group]
with patch('dash.orgs.models.TembaClient1.get_contacts') as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001', name="Ann",urns=['tel:1234'], groups=['000-002'],
fields=dict(state="Lagos", lga="Oyo",gender='Female', born="1990"),
language='eng',
modified_on=datetime(2015, 9, 20, 10, 20, 30, 400000, pytz.utc))]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertTrue('000-001' in seen_uuids)
contact = Contact.objects.get()
self.assertEqual(contact.uuid, '000-001')
self.assertEqual(contact.org, self.nigeria)
self.assertEqual(contact.state, 'R-LAGOS')
self.assertEqual(contact.district, 'R-OYO')
self.assertEqual(contact.gender, 'F')
self.assertEqual(contact.born, 1990)
Contact.fetch_contacts(self.nigeria, after=datetime(2014, 12, 01, 22, 34, 36, 123000, pytz.utc))
self.assertTrue('000-001' in seen_uuids)
# delete the contacts
Contact.objects.all().delete()
group1 = TembaGroup.create(uuid="000-001", name='reporters too', size=10)
group2 = TembaGroup.create(uuid="000-002", name='reporters', size=120)
mock_groups.return_value = [group1, group2]
with patch('dash.orgs.models.TembaClient1.get_contacts') as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001', name="Ann",urns=['tel:1234'], groups=['000-002'],
fields=dict(state="Lagos", lga="Oyo",gender='Female', born="1990"),
language='eng',
modified_on=datetime(2015, 9, 20, 10, 20, 30, 400000, pytz.utc))]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertTrue('000-001' in seen_uuids)
contact = Contact.objects.get()
self.assertEqual(contact.uuid, '000-001')
self.assertEqual(contact.org, self.nigeria)
self.assertEqual(contact.state, 'R-LAGOS')
self.assertEqual(contact.district, 'R-OYO')
self.assertEqual(contact.gender, 'F')
self.assertEqual(contact.born, 1990)
Contact.fetch_contacts(self.nigeria, after=datetime(2014, 12, 01, 22, 34, 36, 123000, pytz.utc))
self.assertTrue('000-001' in seen_uuids)
def test_fetch_contacts(self):
self.nigeria.set_config('reporter_group', 'Reporters')
tz = pytz.timezone('UTC')
with patch.object(timezone,
'now',
return_value=tz.localize(
datetime(2015, 9, 29, 10, 20, 30, 40))):
with patch(
'dash.orgs.models.TembaClient1.get_groups') as mock_groups:
group = TembaGroup.create(uuid="uuid-8",
name='reporters',
size=120)
mock_groups.return_value = [group]
with patch('dash.orgs.models.TembaClient1.get_contacts'
) as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=['000-002'],
fields=dict(state="Lagos",
lga="Oyo",
gender='Female',
born="1990"),
language='eng',
modified_on=datetime(
2015, 9, 20, 10, 20, 30,
400000, pytz.utc))
]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertEqual(seen_uuids, [])
group = TembaGroup.create(uuid="000-002",
name='reporters',
size=120)
mock_groups.return_value = [group]
with patch('dash.orgs.models.TembaClient1.get_contacts'
) as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=['000-002'],
fields=dict(state="Lagos",
lga="Oyo",
gender='Female',
born="1990"),
language='eng',
modified_on=datetime(
2015, 9, 20, 10, 20, 30,
400000, pytz.utc))
]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertTrue('000-001' in seen_uuids)
contact = Contact.objects.get()
self.assertEqual(contact.uuid, '000-001')
self.assertEqual(contact.org, self.nigeria)
self.assertEqual(contact.state, 'R-LAGOS')
self.assertEqual(contact.district, 'R-OYO')
self.assertEqual(contact.gender, 'F')
self.assertEqual(contact.born, 1990)
Contact.fetch_contacts(self.nigeria,
after=datetime(
2014, 12, 01, 22, 34, 36,
123000, pytz.utc))
self.assertTrue('000-001' in seen_uuids)
# delete the contacts
Contact.objects.all().delete()
group1 = TembaGroup.create(uuid="000-001",
name='reporters too',
size=10)
group2 = TembaGroup.create(uuid="000-002",
name='reporters',
size=120)
mock_groups.return_value = [group1, group2]
with patch('dash.orgs.models.TembaClient1.get_contacts'
) as mock_contacts:
mock_contacts.return_value = [
TembaContact.create(uuid='000-001',
name="Ann",
urns=['tel:1234'],
groups=['000-002'],
fields=dict(state="Lagos",
lga="Oyo",
gender='Female',
born="1990"),
language='eng',
modified_on=datetime(
2015, 9, 20, 10, 20, 30,
400000, pytz.utc))
]
seen_uuids = Contact.fetch_contacts(self.nigeria)
self.assertTrue('000-001' in seen_uuids)
contact = Contact.objects.get()
self.assertEqual(contact.uuid, '000-001')
self.assertEqual(contact.org, self.nigeria)
self.assertEqual(contact.state, 'R-LAGOS')
self.assertEqual(contact.district, 'R-OYO')
self.assertEqual(contact.gender, 'F')
self.assertEqual(contact.born, 1990)
Contact.fetch_contacts(self.nigeria,
after=datetime(
2014, 12, 01, 22, 34, 36,
123000, pytz.utc))
self.assertTrue('000-001' in seen_uuids)
