def get_context_data(self, **kwargs):
context = super(PollCRUDL.Read, self).get_context_data(**kwargs)
questions = self.object.get_questions()
pollruns = self.object.get_pollruns(self.request.region, self.request.include_subregions)
# if we're viewing "All Regions" don't include regional only pollruns
if not self.request.region:
pollruns = pollruns.universal()
window = self.request.POST.get("window", self.request.GET.get("window", None))
window = Window[window] if window else Window.last_30_days
window_min, window_max = window.to_range()
pollruns = pollruns.filter(conducted_on__gte=window_min, conducted_on__lt=window_max)
pollruns = pollruns.order_by("conducted_on")
for question in questions:
question.chart_type, question.chart_data = charts.multiple_pollruns(
pollruns, question, self.request.data_regions
)
context["window"] = window
context["window_min"] = datetime_to_ms(window_min)
context["window_max"] = datetime_to_ms(window_max)
context["window_options"] = Window.__members__.values()
context["questions"] = questions
return context
def fetch_results(self, segment=None):
from raven.contrib.django.raven_compat.models import client
cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
if segment and segment.get('location', "") == "District":
cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME
try:
key = CACHE_POLL_RESULTS_KEY % (self.poll.org.pk, self.poll.pk,
self.pk)
if segment:
segment = self.poll.org.substitute_segment(segment)
key += ":" + slugify(unicode(segment))
this_time = datetime.now()
temba_client = self.poll.org.get_temba_client()
client_results = temba_client.get_results(self.ruleset_uuid,
segment=segment)
results = temba_client_flow_results_serializer(client_results)
cache.set(key, {
'time': datetime_to_ms(this_time),
'results': results
}, cache_time)
# delete the open ended cache
cache.delete('open_ended:%d' % self.id)
except: # pragma: no cover
client.captureException()
import traceback
traceback.print_exc()
def fetch_results(self, segment=None):
from raven.contrib.django.raven_compat.models import client
cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
if segment and segment.get('location', "") == "District":
cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME
try:
key = CACHE_POLL_RESULTS_KEY % (self.poll.org.pk, self.poll.pk, self.pk)
if segment:
segment = self.poll.org.substitute_segment(segment)
key += ":" + slugify(unicode(segment))
this_time = datetime.now()
temba_client = self.poll.org.get_temba_client()
client_results = temba_client.get_results(self.ruleset_uuid, segment=segment)
results = temba_client_flow_results_serializer(client_results)
cache.set(key, {'time': datetime_to_ms(this_time), 'results': results}, cache_time)
# delete the open ended cache
cache.delete('open_ended:%d' % self.id)
except: # pragma: no cover
client.captureException()
import traceback
traceback.print_exc()
def fetch_old_sites_count():
import requests, re
from ureport.polls.models import UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
this_time = datetime.now()
linked_sites = list(getattr(settings, 'PREVIOUS_ORG_SITES', []))
old_site_values = []
for site in linked_sites:
count_link = site.get('count_link', "")
if count_link:
try:
response = requests.get(count_link)
response.raise_for_status()
count = int(re.search(r'\d+', response.content).group())
key = "org:%s:reporters:%s" % (site.get('name').lower(),
'old-site')
value = {
'time': datetime_to_ms(this_time),
'results': dict(size=count)
}
old_site_values.append(value)
cache.set(key, value, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
print "Fetch old sites counts took %ss" % (time.time() - start)
return old_site_values
def fetch_flows(org):
start = time.time()
#print "Fetching flows for %s" % org.name
try:
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
this_time = datetime.now()
temba_client = org.get_temba_client()
flows = temba_client.get_flows()
all_flows = dict()
for flow in flows:
if flow.rulesets:
flow_json = dict()
flow_json['uuid'] = flow.uuid
flow_json['created_on'] = flow.created_on.strftime('%Y-%m-%d')
flow_json['name'] = flow.name
flow_json['participants'] = flow.participants
flow_json['runs'] = flow.runs
flow_json['completed_runs'] = flow.completed_runs
flow_json['rulesets'] = [
dict(uuid=elt.uuid, label=elt.label, response_type=elt.response_type) for elt in flow.rulesets]
all_flows[flow.uuid] = flow_json
all_flows_key = CACHE_ORG_FLOWS_KEY % org.pk
cache.set(all_flows_key,
{'time': datetime_to_ms(this_time), 'results': all_flows},
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
def fetch_flows(org, backend=None):
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
logger.info("Fetching flows for %s" % org.name)
if backend:
backends = [backend]
else:
backends = org.backends.filter(is_active=True)
this_time = datetime.now()
org_flows = dict(time=datetime_to_ms(this_time), results=dict())
for backend_obj in backends:
backend = org.get_backend(backend_slug=backend_obj.slug)
try:
all_flows = backend.fetch_flows(org)
org_flows["results"] = all_flows
cache_key = CACHE_ORG_FLOWS_KEY % (org.pk, backend_obj.slug)
cache.set(cache_key, org_flows, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except Exception:
client.captureException()
import traceback
traceback.print_exc()
logger.info("Fetch %s flows took %ss" % (org.name, time.time() - start))
if len(backends):
return org_flows.get("results", dict())
def fetch_old_sites_count():
import requests
import re
from ureport.polls.models import UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
this_time = datetime.now()
linked_sites = list(getattr(settings, "PREVIOUS_ORG_SITES", []))
old_site_values = []
for site in linked_sites:
count_link = site.get("count_link", "")
if count_link:
try:
response = requests.get(count_link)
response.raise_for_status()
count = int(re.search(r"\d+", response.content.decode("utf-8")).group())
key = "org:%s:reporters:%s" % (site.get("name").lower(), "old-site")
value = {"time": datetime_to_ms(this_time), "results": dict(size=count)}
old_site_values.append(value)
cache.set(key, value, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except Exception:
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
logger.info("Fetch old sites counts took %ss" % (time.time() - start))
return old_site_values
def sync_org_contacts(org_id):
"""
Syncs all contacts for the given org
"""
from chatpro.orgs_ext import TaskType
from chatpro.rooms.models import Room
from .models import Contact
org = Org.objects.get(pk=org_id)
logger.info('Starting contact sync task for org #%d' % org.id)
sync_fields = [org.get_chat_name_field()]
sync_groups = [r.uuid for r in Room.get_all(org)]
created, updated, deleted, failed = sync_pull_contacts(org, Contact, fields=sync_fields, groups=sync_groups)
task_result = dict(time=datetime_to_ms(timezone.now()),
counts=dict(created=len(created),
updated=len(updated),
deleted=len(deleted),
failed=len(failed)))
org.set_task_result(TaskType.sync_contacts, task_result)
logger.info("Finished contact sync for org #%d (%d created, %d updated, %d deleted, %d failed)"
% (org.id, len(created), len(updated), len(deleted), len(failed)))
def fetch_old_sites_count():
import requests
import re
from ureport.polls.models import UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
this_time = datetime.now()
linked_sites = list(getattr(settings, "PREVIOUS_ORG_SITES", [])) + list(
getattr(settings, "OTHER_ORG_COUNT_SITES", [])
)
old_site_values = []
for site in linked_sites:
count_link = site.get("count_link", "")
if count_link:
try:
response = requests.get(count_link)
response.raise_for_status()
count = int(re.search(r"\d+", response.content.decode("utf-8")).group())
key = "org:%s:reporters:%s" % (site.get("name").lower(), "old-site")
value = {"time": datetime_to_ms(this_time), "results": dict(size=count)}
old_site_values.append(value)
cache.set(key, value, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except Exception:
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
logger.info("Fetch old sites counts took %ss" % (time.time() - start))
return old_site_values
def fetch_flows(org, backend=None):
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
logger.info("Fetching flows for %s" % org.name)
if backend:
backends = [backend]
else:
backends = org.backends.filter(is_active=True)
this_time = datetime.now()
org_flows = dict(time=datetime_to_ms(this_time), results=dict())
for backend_obj in backends:
backend = org.get_backend(backend_slug=backend_obj.slug)
try:
all_flows = backend.fetch_flows(org)
org_flows["results"] = all_flows
cache_key = CACHE_ORG_FLOWS_KEY % (org.pk, backend_obj.slug)
cache.set(cache_key, org_flows, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except Exception:
client.captureException()
import traceback
traceback.print_exc()
logger.info("Fetch %s flows took %ss" % (org.name, time.time() - start))
if len(backends):
return org_flows.get("results", dict())
def sync_org_contacts(org_id):
"""
Syncs all contacts for the given org
"""
from tracpro.groups.models import Region, Group
from tracpro.orgs_ext.constants import TaskType
from .models import Contact
org = Org.objects.get(pk=org_id)
logger.info('Starting contact sync task for org #%d' % org.id)
sync_groups = [r.uuid for r in Region.get_all(org)] + [g.uuid for g in Group.get_all(org)]
most_recent_contact = Contact.objects.by_org(org).active().exclude(temba_modified_on=None)
most_recent_contact = most_recent_contact.order_by('-temba_modified_on').first()
if most_recent_contact:
last_time = most_recent_contact.temba_modified_on
else:
last_time = None
created, updated, deleted, failed = sync_pull_contacts(
org, Contact, fields=(), groups=sync_groups, last_time=last_time,
delete_blocked=True)
task_result = dict(time=datetime_to_ms(timezone.now()),
counts=dict(created=len(created),
updated=len(updated),
deleted=len(deleted),
failed=len(failed)))
org.set_task_result(TaskType.sync_contacts, task_result)
logger.info("Finished contact sync for org #%d (%d created, "
"%d updated, %d deleted, %d failed)" %
(org.id, len(created), len(updated), len(deleted), len(failed)))
def fetch_reporter_group(org):
start = time.time()
print "Fetching reporter group for %s" % org.name
try:
from ureport.polls.models import CACHE_ORG_REPORTER_GROUP_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
this_time = datetime.now()
reporter_group = org.get_config('reporter_group')
if reporter_group:
temba_client = org.get_temba_client()
groups = temba_client.get_groups(name=reporter_group)
key = CACHE_ORG_REPORTER_GROUP_KEY % (org.pk, slugify(unicode(reporter_group)))
group_dict = dict()
if groups:
group = groups[0]
group_dict = dict(size=group.size, name=group.name, uuid=group.uuid)
cache.set(key,
{'time': datetime_to_ms(this_time), 'results': group_dict},
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
print "Fetch %s reporter group took %ss" % (org.name, time.time() - start)
def sync_org_contacts(org_id):
"""
Syncs all contacts for the given org
"""
from tracpro.groups.models import Region, Group
from tracpro.orgs_ext import TaskType
from .models import Contact
org = Org.objects.get(pk=org_id)
logger.info('Starting contact sync task for org #%d' % org.id)
sync_groups = [r.uuid for r in Region.get_all(org)] + [g.uuid for g in Group.get_all(org)]
created, updated, deleted, failed = sync_pull_contacts(org, Contact, fields=(), groups=sync_groups)
task_result = dict(time=datetime_to_ms(timezone.now()),
counts=dict(created=len(created),
updated=len(updated),
deleted=len(deleted),
failed=len(failed)))
org.set_task_result(TaskType.sync_contacts, task_result)
logger.info("Finished contact sync for org #%d (%d created, %d updated, %d deleted, %d failed)"
% (org.id, len(created), len(updated), len(deleted), len(failed)))
def fetch_old_sites_count():
import requests, re
from ureport.polls.models import UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
this_time = datetime.now()
linked_sites = list(getattr(settings, 'PREVIOUS_ORG_SITES', []))
for site in linked_sites:
count_link = site.get('count_link', "")
if count_link:
try:
response = requests.get(count_link)
response.raise_for_status()
count = int(re.search(r'\d+', response.content).group())
key = "org:%s:reporters:%s" % (site.get('name').lower(), 'old-site')
cache.set(key,
{'time': datetime_to_ms(this_time), 'results': dict(size=count)},
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
print "Fetch old sites counts took %ss" % (time.time() - start)
def fetch_contact_field_results(org, contact_field, segment):
from ureport.polls.models import CACHE_ORG_FIELD_DATA_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
from ureport.polls.models import UREPORT_RUN_FETCHED_DATA_CACHE_TIME
start = time.time()
print "Fetching %s for %s with segment %s" % (contact_field, org.name, segment)
cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
if segment and segment.get('location', "") == "District":
cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME
try:
segment = substitute_segment(org, segment)
this_time = datetime.now()
temba_client = org.get_temba_client()
client_results = temba_client.get_results(contact_field=contact_field, segment=segment)
results_data = temba_client_flow_results_serializer(client_results)
cleaned_results_data = results_data
print "Fetch took %ss" % (time.time() - start)
key = CACHE_ORG_FIELD_DATA_KEY % (org.pk, slugify(unicode(contact_field)), slugify(unicode(segment)))
cache.set(key, {'time': datetime_to_ms(this_time), 'results': cleaned_results_data}, cache_time)
except:
client.captureException()
import traceback
traceback.print_exc()
def org_task(self, org):
"""
Fetches new and modified flow runs for the given org and creates/updates
poll responses.
"""
from tracpro.orgs_ext.constants import TaskType
from tracpro.polls.models import Poll, PollRun, Response
client = org.get_temba_client()
redis_connection = get_redis_connection()
last_time_key = LAST_FETCHED_RUN_TIME_KEY % org.pk
last_time = redis_connection.get(last_time_key)
if last_time is not None:
last_time = parse_iso8601(last_time)
else:
newest_runs = Response.objects.filter(
pollrun__poll__org=org).order_by('-created_on')
newest_runs = newest_runs.exclude(
pollrun__pollrun_type=PollRun.TYPE_SPOOFED)
newest_run = newest_runs.first()
last_time = newest_run.created_on if newest_run else None
until = timezone.now()
total_runs = 0
for poll in Poll.objects.active().by_org(org):
poll_runs = client.get_runs(flows=[poll.flow_uuid],
after=last_time,
before=until)
total_runs += len(poll_runs)
# convert flow runs into poll responses
for run in poll_runs:
try:
Response.from_run(org, run, poll=poll)
except ValueError as e:
logger.error("Unable to save run #%d due to error: %s" %
(run.id, e.message))
continue
logger.info("Fetched %d new and updated runs for org #%d (since=%s)" %
(total_runs, org.id,
format_iso8601(last_time) if last_time else 'Never'))
task_result = dict(time=datetime_to_ms(timezone.now()),
counts=dict(fetched=total_runs))
org.set_task_result(TaskType.fetch_runs, task_result)
redis_connection.set(last_time_key, format_iso8601(until))
def fetch_shared_sites_count():
import requests
this_time = datetime.now()
try:
response = requests.get("https://ureport.in/shared_sites_count/")
response.raise_for_status()
value = {"time": datetime_to_ms(this_time), "results": response.json()}
cache.set("shared_sites", value, None)
return value["results"]
except Exception:
import traceback
traceback.print_exc()
def fetch_org_runs(org_id):
"""
Fetches new and modified flow runs for the given org and creates/updates
poll responses.
"""
from tracpro.orgs_ext.constants import TaskType
from tracpro.polls.models import Poll, PollRun, Response
org = Org.objects.get(pk=org_id)
client = org.get_temba_client()
redis_connection = get_redis_connection()
last_time_key = LAST_FETCHED_RUN_TIME_KEY % org.pk
last_time = redis_connection.get(last_time_key)
if last_time is not None:
last_time = parse_iso8601(last_time)
else:
newest_runs = Response.objects.filter(pollrun__poll__org=org).order_by("-created_on")
newest_runs = newest_runs.exclude(pollrun__pollrun_type=PollRun.TYPE_SPOOFED)
newest_run = newest_runs.first()
last_time = newest_run.created_on if newest_run else None
until = timezone.now()
total_runs = 0
for poll in Poll.get_all(org):
poll_runs = client.get_runs(flows=[poll.flow_uuid], after=last_time, before=until)
total_runs += len(poll_runs)
# convert flow runs into poll responses
for run in poll_runs:
try:
Response.from_run(org, run, poll=poll)
except ValueError as e:
logger.error("Unable to save run #%d due to error: %s" % (run.id, e.message))
continue
logger.info(
"Fetched %d new and updated runs for org #%d (since=%s)"
% (total_runs, org.id, format_iso8601(last_time) if last_time else "Never")
)
task_result = dict(time=datetime_to_ms(timezone.now()), counts=dict(fetched=total_runs))
org.set_task_result(TaskType.fetch_runs, task_result)
redis_connection.set(last_time_key, format_iso8601(until))
def process_new_org_unsolicited(org):
"""
Processes new unsolicited messages for an org in RapidPro
"""
from .models import Message
client = org.get_temba_client()
# when was this task last run?
last_result = org.get_task_result(TaskType.label_messages)
if last_result:
last_time = ms_to_datetime(last_result['time'])
else:
# if first time (or Redis bombed...) then we'll fetch back to 3 hours ago
last_time = timezone.now() - timedelta(hours=3)
this_time = timezone.now()
num_messages = 0
num_labelled = 0
# grab all un-processed unsolicited messages
pager = client.pager()
while True:
messages = client.get_messages(direction='I',
_types=['I'],
archived=False,
after=last_time,
before=this_time,
pager=pager)
num_messages += len(messages)
num_labelled += Message.process_unsolicited(org, messages)
if not pager.has_more():
break
print "Processed %d new unsolicited messages and labelled %d" % (
num_messages, num_labelled)
org.set_task_result(
TaskType.label_messages, {
'time': datetime_to_ms(this_time),
'counts': {
'messages': num_messages,
'labelled': num_labelled
}
})
def fetch_flows(org):
start = time.time()
print "Fetching flows for %s" % org.name
this_time = datetime.now()
org_flows = dict(time=datetime_to_ms(this_time), results=dict())
try:
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
temba_client = org.get_temba_client()
flows = temba_client.get_flows()
all_flows = dict()
for flow in flows:
if flow.rulesets:
flow_json = dict()
flow_json['uuid'] = flow.uuid
flow_json['date_hint'] = flow.created_on.strftime('%Y-%m-%d')
flow_json['created_on'] = datetime_to_json_date(
flow.created_on)
flow_json['name'] = flow.name
flow_json['runs'] = flow.runs
flow_json['archived'] = flow.archived
flow_json['completed_runs'] = flow.completed_runs
flow_json['rulesets'] = [
dict(uuid=elt.uuid,
label=elt.label,
response_type=elt.response_type)
for elt in flow.rulesets
]
all_flows[flow.uuid] = flow_json
all_flows_key = CACHE_ORG_FLOWS_KEY % org.pk
org_flows['results'] = all_flows
cache.set(all_flows_key, org_flows,
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
print "Fetch %s flows took %ss" % (org.name, time.time() - start)
return org_flows.get('results')
def build_boundaries(self):
this_time = datetime.now()
temba_client = self.get_temba_client()
client_boundaries = temba_client.get_boundaries()
# we now build our cached versions of level 1 (all states) and level 2
# (all districts for each state) geojson
start_level = []
other_levels_by_parent = dict()
for boundary in client_boundaries:
if boundary.level == BOUNDARY_START_LEVEL:
start_level.append(boundary)
elif boundary.level <= BOUNDARY_END_LEVEL and boundary.parent:
osm_id = boundary.parent
if osm_id not in other_levels_by_parent:
other_levels_by_parent[osm_id] = []
districts = other_levels_by_parent[osm_id]
districts.append(boundary)
# mini function to convert a list of boundary objects to geojson
def to_geojson(boundary_list):
features = [
dict(type='Feature',
geometry=dict(type=b.geometry.type,
coordinates=b.geometry.coordinates),
properties=dict(name=b.name, id=b.boundary,
level=b.level)) for b in boundary_list
]
return dict(type='FeatureCollection', features=features)
boundaries = dict()
boundaries[BOUNDARY_LEVEL_START_KEY %
self.id] = to_geojson(start_level)
for parent_id in other_levels_by_parent.keys():
boundaries[BOUNDARY_LEVEL_END_KEY %
(self.id, parent_id)] = to_geojson(
other_levels_by_parent[parent_id])
key = BOUNDARY_CACHE_KEY % self.pk
value = {'time': datetime_to_ms(this_time), 'results': boundaries}
cache.set(key, value, BOUNDARY_CACHE_TIME)
return boundaries
def build_boundaries(self):
this_time = datetime.now()
temba_client = self.get_temba_client()
client_boundaries = temba_client.get_boundaries()
# we now build our cached versions of level 1 (all states) and level 2
# (all districts for each state) geojson
states = []
districts_by_state = dict()
for boundary in client_boundaries:
if boundary.level == STATE:
states.append(boundary)
elif boundary.level == DISTRICT:
osm_id = boundary.parent
if osm_id not in districts_by_state:
districts_by_state[osm_id] = []
districts = districts_by_state[osm_id]
districts.append(boundary)
# mini function to convert a list of boundary objects to geojson
def to_geojson(boundary_list):
features = [dict(type='Feature',
geometry=dict(type=b.geometry.type,
coordinates=b.geometry.coordinates),
properties=dict(name=b.name, id=b.boundary, level=b.level))
for b in boundary_list]
return dict(type='FeatureCollection', features=features)
boundaries = dict()
boundaries[BOUNDARY_LEVEL_1_KEY % self.id] = to_geojson(states)
for state_id in districts_by_state.keys():
boundaries[BOUNDARY_LEVEL_2_KEY % (self.id, state_id)] = to_geojson(
districts_by_state[state_id])
key = BOUNDARY_CACHE_KEY % self.pk
value = {'time': datetime_to_ms(this_time), 'results': boundaries}
cache.set(key, value, BOUNDARY_CACHE_TIME)
return boundaries
def fetch_flows(org):
start = time.time()
print "Fetching flows for %s" % org.name
this_time = datetime.now()
org_flows = dict(time=datetime_to_ms(this_time), results=dict())
try:
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
temba_client = org.get_temba_client()
flows = temba_client.get_flows()
all_flows = dict()
for flow in flows:
if flow.rulesets:
flow_json = dict()
flow_json['uuid'] = flow.uuid
flow_json['date_hint'] = flow.created_on.strftime('%Y-%m-%d')
flow_json['created_on'] = datetime_to_json_date(flow.created_on)
flow_json['name'] = flow.name
flow_json['runs'] = flow.runs
flow_json['archived'] = flow.archived
flow_json['completed_runs'] = flow.completed_runs
flow_json['rulesets'] = [
dict(uuid=elt.uuid, label=elt.label, response_type=elt.response_type) for elt in flow.rulesets]
all_flows[flow.uuid] = flow_json
all_flows_key = CACHE_ORG_FLOWS_KEY % org.pk
org_flows['results'] = all_flows
cache.set(all_flows_key, org_flows, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
print "Fetch %s flows took %ss" % (org.name, time.time() - start)
return org_flows.get('results')
def sync(self, org):
recent_contacts = Contact.objects.by_org(org).active()
recent_contacts = recent_contacts.exclude(temba_modified_on=None)
recent_contacts = recent_contacts.order_by('-temba_modified_on')
most_recent = recent_contacts.first()
sync_regions = [r.uuid for r in Region.get_all(org)]
sync_groups = [g.uuid for g in Group.get_all(org)]
created, updated, deleted, failed = sync_pull_contacts(
org=org, contact_class=Contact, fields=(), delete_blocked=True,
groups=sync_regions + sync_groups,
last_time=most_recent.temba_modified_on if most_recent else None)
org.set_task_result(TaskType.sync_contacts, {
'time': datetime_to_ms(timezone.now()),
'counts': {
'created': len(created),
'updated': len(updated),
'deleted': len(deleted),
'failed': len(failed),
},
})
def process_new_org_unsolicited(org):
"""
Processes new unsolicited messages for an org in RapidPro
"""
from .models import Message
client = org.get_temba_client()
# when was this task last run?
last_result = org.get_task_result(TaskType.label_messages)
if last_result:
last_time = ms_to_datetime(last_result['time'])
else:
# if first time (or Redis bombed...) then we'll fetch back to 3 hours ago
last_time = timezone.now() - timedelta(hours=3)
this_time = timezone.now()
num_messages = 0
num_labelled = 0
# grab all un-processed unsolicited messages
pager = client.pager()
while True:
messages = client.get_messages(direction='I', _types=['I'], archived=False,
after=last_time, before=this_time, pager=pager)
num_messages += len(messages)
num_labelled += Message.process_unsolicited(org, messages)
if not pager.has_more():
break
print "Processed %d new unsolicited messages and labelled %d" % (num_messages, num_labelled)
org.set_task_result(TaskType.label_messages, {'time': datetime_to_ms(this_time),
'counts': {'messages': num_messages, 'labelled': num_labelled}})
def default(self, obj):
if isinstance(obj, datetime.datetime):
return datetime_to_ms(obj)
elif isinstance(obj, Decimal):
return float(obj)
return json.JSONEncoder.default(self, obj)
def default(self, obj):
if isinstance(obj, datetime.datetime):
return datetime_to_ms(obj)
elif isinstance(obj, Decimal):
return float(obj)
return json.JSONEncoder.default(self, obj)
