def pre_save(self, obj):
obj = super(PollCRUDL.Create, self).pre_save(obj)
org = self.request.org
obj.org = org
obj.backend = org.backends.filter(is_active=True).first()
now = timezone.now()
five_minutes_ago = now - timedelta(minutes=5)
similar_poll = Poll.objects.filter(
org=obj.org,
flow_uuid="",
backend=obj.backend,
is_active=True,
created_on__gte=five_minutes_ago).first()
if similar_poll:
obj = similar_poll
flow = obj.get_flow()
date = flow.get("created_on", None)
if date:
flow_date = json_date_to_datetime(date)
else:
flow_date = timezone.now()
obj.poll_date = flow_date
return obj
def populate_poll_poll_date(apps, schema_editor):
Poll = apps.get_model("polls", "Poll")
Org = apps.get_model("orgs", "Org")
agent = getattr(settings, "SITE_API_USER_AGENT", None)
host = settings.SITE_API_HOST
for org in Org.objects.all():
temba_client = TembaClient(host, org.api_token, user_agent=agent)
api_flows = temba_client.get_flows()
flows_date = dict()
for flow in api_flows:
flows_date[flow.uuid] = datetime_to_json_date(flow.created_on)
for poll in Poll.objects.filter(org=org):
json_date = flows_date.get(poll.flow_uuid, None)
if json_date:
date = json_date_to_datetime(json_date)
else:
logger.info(
"using created_on for flow_date on poll with id %s" %
poll.pk)
date = poll.created_on
poll.poll_date = date
poll.save()
def get_poll_sync_status(obj):
if obj.has_synced:
last_synced = cache.get(Poll.POLL_RESULTS_LAST_SYNC_TIME_CACHE_KEY % (obj.org.pk, obj.flow_uuid), None)
if last_synced:
return timesince(json_date_to_datetime(last_synced))
# we know we synced do not check the the progress since that is slow
return "Synced 100%"
sync_progress = obj.get_sync_progress()
return "Syncing... {0:.1f}%".format(sync_progress)
def clean(self):
cleaned_data = self.cleaned_data
poll_date = cleaned_data.get("poll_date")
poll_end_date = cleaned_data.get("poll_end_date")
flows = self.org.get_flows(self.flow.backend)
flow = flows.get(self.flow.flow_uuid)
if not poll_date and flow:
date = flow.get("created_on", None)
if date:
poll_date = json_date_to_datetime(date)
if not poll_date:
poll_date = timezone.now()
cleaned_data["poll_date"] = poll_date
cleaned_data["poll_end_date"] = poll_end_date
return cleaned_data
def pull_results(self,
poll,
modified_after,
modified_before,
progress_callback=None):
org = poll.org
r = get_redis_connection()
key = Poll.POLL_PULL_RESULTS_TASK_LOCK % (org.pk, poll.flow_uuid)
stats_dict = dict(
num_val_created=0,
num_val_updated=0,
num_val_ignored=0,
num_path_created=0,
num_path_updated=0,
num_path_ignored=0,
num_synced=0,
)
if r.get(key):
logger.info(
"Skipping pulling results for poll #%d on org #%d as it is still running"
% (poll.pk, org.pk))
else:
with r.lock(key, timeout=Poll.POLL_SYNC_LOCK_TIMEOUT):
lock_expiration = time.time(
) + 0.8 * Poll.POLL_SYNC_LOCK_TIMEOUT
client = self._get_client(org, 2)
questions_uuids = poll.get_question_uuids()
# ignore the TaskState time and use the time we stored in redis
(
after,
before,
latest_synced_obj_time,
batches_latest,
resume_cursor,
pull_after_delete,
) = poll.get_pull_cached_params()
if pull_after_delete is not None:
after = None
latest_synced_obj_time = None
batches_latest = None
resume_cursor = None
poll.delete_poll_results()
pull_refresh_from_archives.apply_async((poll.pk, ),
queue="sync")
if resume_cursor is None:
before = datetime_to_json_date(timezone.now())
after = latest_synced_obj_time
start = time.time()
logger.info("Start fetching runs for poll #%d on org #%d" %
(poll.pk, org.pk))
poll_runs_query = client.get_runs(flow=poll.flow_uuid,
after=after,
before=before)
fetches = poll_runs_query.iterfetches(
retry_on_rate_exceed=True, resume_cursor=resume_cursor)
try:
fetch_start = time.time()
for fetch in fetches:
logger.info("RapidPro API fetch for poll #%d "
"on org #%d %d - %d took %ds" % (
poll.pk,
org.pk,
stats_dict["num_synced"],
stats_dict["num_synced"] + len(fetch),
time.time() - fetch_start,
))
contacts_map, poll_results_map, poll_results_to_save_map = self._initiate_lookup_maps(
fetch, org, poll)
for temba_run in fetch:
if batches_latest is None or temba_run.modified_on > json_date_to_datetime(
batches_latest):
batches_latest = datetime_to_json_date(
temba_run.modified_on.replace(
tzinfo=pytz.utc))
contact_obj = contacts_map.get(
temba_run.contact.uuid, None)
self._process_run_poll_results(
org,
questions_uuids,
temba_run,
contact_obj,
poll_results_map,
poll_results_to_save_map,
stats_dict,
)
stats_dict["num_synced"] += len(fetch)
if progress_callback:
progress_callback(stats_dict["num_synced"])
self._save_new_poll_results_to_database(
poll_results_to_save_map)
logger.info(
"Processed fetch of %d - %d "
"runs for poll #%d on org #%d" %
(stats_dict["num_synced"] - len(fetch),
stats_dict["num_synced"], poll.pk, org.pk))
fetch_start = time.time()
logger.info("=" * 40)
if (stats_dict["num_synced"] >=
Poll.POLL_RESULTS_MAX_SYNC_RUNS
or time.time() > lock_expiration):
poll.rebuild_poll_results_counts()
cursor = fetches.get_cursor()
self._mark_poll_results_sync_paused(
org, poll, cursor, after, before,
batches_latest)
logger.info(
"Break pull results for poll #%d on org #%d in %ds, "
" Times: after= %s, before= %s, batch_latest= %s, sync_latest= %s"
" Objects: created %d, updated %d, ignored %d. "
"Before cursor %s" % (
poll.pk,
org.pk,
time.time() - start,
after,
before,
batches_latest,
latest_synced_obj_time,
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
cursor,
))
return (
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
stats_dict["num_path_created"],
stats_dict["num_path_updated"],
stats_dict["num_path_ignored"],
)
except TembaRateExceededError:
poll.rebuild_poll_results_counts()
cursor = fetches.get_cursor()
self._mark_poll_results_sync_paused(
org, poll, cursor, after, before, batches_latest)
logger.info(
"Break pull results for poll #%d on org #%d in %ds, "
" Times: after= %s, before= %s, batch_latest= %s, sync_latest= %s"
" Objects: created %d, updated %d, ignored %d. "
"Before cursor %s" % (
poll.pk,
org.pk,
time.time() - start,
after,
before,
batches_latest,
latest_synced_obj_time,
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
cursor,
))
return (
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
stats_dict["num_path_created"],
stats_dict["num_path_updated"],
stats_dict["num_path_ignored"],
)
if batches_latest is not None and (
latest_synced_obj_time is None
or json_date_to_datetime(latest_synced_obj_time) <=
json_date_to_datetime(batches_latest)):
latest_synced_obj_time = batches_latest
self._mark_poll_results_sync_completed(poll, org,
latest_synced_obj_time)
# from django.db import connection as db_connection, reset_queries
# slowest_queries = sorted(db_connection.queries, key=lambda q: q['time'], reverse=True)[:10]
# for q in slowest_queries:
# print "=" * 60
# print "\n\n\n"
# print "%s -- %s" % (q['time'], q['sql'])
# reset_queries()
logger.info(
"Finished pulling results for poll #%d on org #%d runs in %ds, "
"Times: sync_latest= %s,"
"Objects: created %d, updated %d, ignored %d" % (
poll.pk,
org.pk,
time.time() - start,
latest_synced_obj_time,
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
))
return (
stats_dict["num_val_created"],
stats_dict["num_val_updated"],
stats_dict["num_val_ignored"],
stats_dict["num_path_created"],
stats_dict["num_path_updated"],
stats_dict["num_path_ignored"],
)
def local_kwargs(self, org, remote):
from rtm.utils import json_date_to_datetime
reporter_group = org.get_config("%s.reporter_group" %
self.backend.slug,
default="")
contact_groups_names = [group.name.lower() for group in remote.groups]
if not reporter_group.lower() in contact_groups_names:
return None
org_state_boundaries_data, org_district_boundaries_data, org_ward_boundaries_data = self.get_boundaries_data(
org)
contact_fields = self.get_contact_fields(org)
state = ""
district = ""
ward = ""
state_field = org.get_config("%s.state_label" % self.backend.slug,
default="")
if state_field:
state_field = state_field.lower()
if org.get_config("common.is_global"):
state_name = remote.fields.get(contact_fields.get(state_field),
None)
if state_name:
state = state_name
else:
state_path = remote.fields.get(contact_fields.get(state_field),
None)
if state_path:
state_name = state_path.split(" > ")[-1]
state_name = state_name.lower()
state = org_state_boundaries_data.get(state_name, "")
district_field = org.get_config("%s.district_label" %
self.backend.slug,
default="")
if district_field:
district_field = district_field.lower()
district_path = remote.fields.get(
contact_fields.get(district_field), None)
if district_path:
district_name = district_path.split(" > ")[-1]
district_name = district_name.lower()
district = org_district_boundaries_data.get(
state, dict()).get(district_name, "")
ward_field = org.get_config("%s.ward_label" %
self.backend.slug,
default="")
if ward_field:
ward_field = ward_field.lower()
ward_path = remote.fields.get(
contact_fields.get(ward_field), None)
if ward_path:
ward_name = ward_path.split(" > ")[-1]
ward_name = ward_name.lower()
ward = org_ward_boundaries_data.get(district,
dict()).get(
ward_name, "")
registered_on = None
registration_field = org.get_config("%s.registration_label" %
self.backend.slug,
default="")
if registration_field:
registration_field = registration_field.lower()
registered_on = remote.fields.get(
contact_fields.get(registration_field), None)
if registered_on:
registered_on = json_date_to_datetime(registered_on)
if not registered_on:
# default to created_on to avoid null in the PG triggers
registered_on = remote.created_on
occupation = ""
occupation_field = org.get_config("%s.occupation_label" %
self.backend.slug,
default="")
if occupation_field:
occupation_field = occupation_field.lower()
occupation = remote.fields.get(
contact_fields.get(occupation_field), "")
if not occupation:
occupation = ""
born = 0
born_field = org.get_config("%s.born_label" % self.backend.slug,
default="")
if born_field:
born_field = born_field.lower()
try:
born = int(remote.fields.get(contact_fields.get(born_field),
0))
# support only positive django integer field valid values
if born < 0 or born > 2147483647:
born = 0
except ValueError:
pass
except TypeError:
pass
gender = ""
gender_field = org.get_config("%s.gender_label" % self.backend.slug,
default="")
female_label = org.get_config("%s.female_label" % self.backend.slug,
default="")
male_label = org.get_config("%s.male_label" % self.backend.slug,
default="")
extra_gender = org.get_config("common.has_extra_gender", default=False)
if gender_field:
gender_field = gender_field.lower()
gender = remote.fields.get(contact_fields.get(gender_field), "")
if gender and gender.lower() == female_label.lower():
gender = self.model.FEMALE
elif gender and gender.lower() == male_label.lower():
gender = self.model.MALE
elif gender and extra_gender:
gender = self.model.OTHER
else:
gender = ""
return {
"backend": self.backend,
"org": org,
"uuid": remote.uuid,
"gender": gender,
"born": born,
"occupation": occupation,
"registered_on": registered_on,
"state": state,
"district": district,
"ward": ward,
}