def get_location_response_rate_series(cls, org, time_filter):
now = timezone.now()
year_ago = now - timedelta(days=365)
start = year_ago.replace(day=1)
top_boundaries = Boundary.get_org_top_level_boundaries_name(org)
output_data = []
for osm_id, name in top_boundaries.items():
boundary_ids = list(
Boundary.objects.filter(org=org).filter(
Q(osm_id=osm_id) | Q(parent__osm_id=osm_id)
| Q(parent__parent__osm_id=osm_id)).values_list("pk",
flat=True))
polled_stats = (PollStats.objects.filter(
org=org, date__gte=start,
location_id__in=boundary_ids).values("date").annotate(
Sum("count")))
responded_stats = (PollStats.objects.filter(
org=org, date__gte=start,
location_id__in=boundary_ids).exclude(
category=None).values("date").annotate(Sum("count")))
series = PollStats.get_response_rate_data(polled_stats,
responded_stats,
time_filter)
output_data.append(dict(name=name, osm_id=osm_id, data=series))
return output_data
def get_context_data(self, **kwargs):
context = super(PollContextMixin, self).get_context_data(**kwargs)
org = self.request.org
context["org"] = org
context["gender_stats"] = org.get_gender_stats()
context["age_stats"] = org.get_age_stats()
context["states"] = [dict(id=k, name=v) for k, v in Boundary.get_org_top_level_boundaries_name(org).items()]
main_poll = self.derive_main_poll()
context["latest_poll"] = main_poll
if main_poll:
top_question = main_poll.get_questions().first()
context["top_question"] = top_question
if top_question:
gender_stats = top_question.get_gender_stats()
total_gender = 0
for elt in gender_stats:
total_gender += elt["set"]
gender_stats_dict = {
elt["label"].lower(): dict(
count=elt["set"], percentage=int(round(elt["set"] * 100 / float(total_gender)))
)
for elt in gender_stats
if total_gender
}
context["gender_stats"] = gender_stats_dict
age_stats = top_question.get_age_stats()
total_age = 0
for elt in age_stats:
total_age += elt["set"]
context["age_stats"] = [
dict(name=elt["label"], y=int(round(elt["set"] * 100 / float(total_age))))
for elt in age_stats
if total_age
]
context["locations_stats"] = top_question.get_location_stats()
context["categories"] = (
Category.objects.filter(org=org, is_active=True)
.prefetch_related(Prefetch("polls", queryset=Poll.objects.filter(is_active=True).order_by("-poll_date")))
.order_by("name")
)
context["polls"] = Poll.get_public_polls(org=org).order_by("-poll_date")
context["related_stories"] = []
if main_poll:
related_stories = Story.objects.filter(org=org, is_active=True, category=main_poll.category)
related_stories = related_stories.order_by("-featured", "-created_on")
context["related_stories"] = related_stories
context["main_stories"] = Story.objects.filter(org=org, featured=True, is_active=True).order_by("-created_on")
return context
def refresh_caches(org, caches):
if OrgCache.boundaries in caches:
Boundary.fetch_boundaries(org)
if OrgCache.main_polls in caches:
refresh_main_poll.delay(org.pk)
if OrgCache.brick_polls in caches:
refresh_brick_polls.delay(org.pk)
if OrgCache.other_polls in caches:
refresh_other_polls.delay(org.pk)
if OrgCache.flows in caches:
refresh_org_flows.delay(org.pk)
def get_sign_up_rate_location(org, time_filter):
now = timezone.now()
year_ago = now - timedelta(days=365)
start = year_ago.replace(day=1)
tz = pytz.timezone("UTC")
org_contacts_counts = get_org_contacts_counts(org)
registered_on_counts = {k[17:]: v for k, v in org_contacts_counts.items() if k.startswith("registered_state")}
top_boundaries = Boundary.get_org_top_level_boundaries_name(org)
keys = get_last_months(months_num=time_filter)
output_data = []
for osm_id, name in top_boundaries.items():
interval_dict = defaultdict(int)
for date_key, date_count in registered_on_counts.items():
if date_key.endswith(f":{osm_id.upper()}"):
date_key = date_key[:10]
else:
continue
parsed_time = tz.localize(datetime.strptime(date_key, "%Y-%m-%d")).replace(
day=1, hour=0, minute=0, second=0, microsecond=0
)
if parsed_time > start:
interval_dict[str(parsed_time.date())] += date_count
data = dict()
for key in keys:
data[key] = interval_dict[key]
output_data.append(dict(name=name, osm_id=osm_id, data=data))
return output_data
def fetch_contacts_task(org_id=None, fetch_all=False):
r = get_redis_connection()
key = 'fetch_contacts'
lock_timeout = 3600
if org_id:
key = 'fetch_contacts:%d' % org_id
lock_timeout = 300
if not r.get(key):
with r.lock(key, timeout=lock_timeout):
active_orgs = Org.objects.filter(is_active=True)
if org_id:
active_orgs = Org.objects.filter(pk=org_id)
for org in active_orgs:
start = time.time()
last_fetched_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % org.id
after = cache.get(last_fetched_key, None)
if after:
after = json_date_to_datetime(after)
if fetch_all:
after = None
try:
if after is None:
Boundary.fetch_boundaries(org)
ContactField.fetch_contact_fields(org)
Boundary.get_boundaries(org)
ContactField.get_contact_fields(org)
Contact.fetch_contacts(org, after=after)
print "Task: fetch_contacts for %s took %ss" % (org.name, time.time() - start)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception("Error fetching contacts: %s" % str(e))
def get_context_data(self, **kwargs):
context = super(UreportersView, self).get_context_data(**kwargs)
org = self.request.org
context["org"] = org
translation.activate(org.language)
# remove the first option '' from calender.month_abbr
context["months"] = [
six.text_type(_("%s")) % m for m in calendar.month_abbr
][1:]
context["states"] = sorted(
[
dict(id=k, name=v) for k, v in
Boundary.get_org_top_level_boundaries_name(org).items()
],
key=lambda c: c["name"],
)
context["gender_stats"] = org.get_gender_stats()
context["age_stats"] = json.loads(org.get_age_stats())
context["registration_stats"] = org.get_registration_stats()
context["occupation_stats"] = org.get_occupation_stats()
context["reporters"] = org.get_reporters_count()
context["main_stories"] = Story.objects.filter(
org=org, featured=True, is_active=True).order_by("-created_on")
# global counter
context["global_counter"] = get_global_count()
context["average_response_rate"] = PollStats.get_average_response_rate(
org)
context["data_time_filters"] = [
dict(time_filter_number=key, label=str(val))
for key, val in PollStats.DATA_TIME_FILTERS.items()
]
backend_options = org.backends.filter(is_active=True).values_list(
"slug", flat=True)
show_maps = reduce(operator.or_, [
bool(org.get_config("%s.state_label" % option))
for option in backend_options
], False)
context["data_segments"] = [
dict(segment_type=key, label=str(val))
for key, val in PollStats.DATA_SEGMENTS.items()
if key != "location" or show_maps
]
context["data_metrics"] = [
dict(slug=key, title=str(val))
for key, val in PollStats.DATA_METRICS.items()
]
return context
def pull_boundaries(self, org):
if org.get_config('is_global'):
incoming_objects = Boundary.build_global_boundaries()
else:
client = self._get_client(org, 1)
incoming_objects = client.get_boundaries()
return sync_local_to_set(org, BoundarySyncer(), incoming_objects)
def pull_boundaries(self, org):
if org.get_config('is_global'):
incoming_objects = Boundary.build_global_boundaries()
else:
client = self._get_client(org, 1)
incoming_objects = client.get_boundaries()
return sync_local_to_set(org, BoundarySyncer(), incoming_objects)
def pull_boundaries(self, org):
if org.get_config("common.is_global"):
incoming_objects = Boundary.build_global_boundaries()
else:
client = self._get_client(org, 2)
incoming_objects = client.get_boundaries(geometry=True).all()
return sync_local_to_set(org, BoundarySyncer(backend=self.backend), incoming_objects)
def pull_boundaries(self, org):
if org.get_config('common.is_global'):
incoming_objects = Boundary.build_global_boundaries()
else:
client = self._get_client(org, 2)
incoming_objects = client.get_boundaries(geometry=True).all()
return sync_local_to_set(org, BoundarySyncer(backend=self.backend), incoming_objects)
def get_regions_stats(org):
org_contacts_counts = get_org_contacts_counts(org)
boundaries_name = Boundary.get_org_top_level_boundaries_name(org)
boundaries_stats = {k[6:]: v for k, v in org_contacts_counts.iteritems() if len(k) > 7 and k.startswith('state')}
regions_stats = sorted([dict(name=boundaries_name[k], count=v) for k, v in boundaries_stats.iteritems()
if k and k in boundaries_name], key=lambda i: i['count'], reverse=True)
return regions_stats
def refresh_caches(org, caches):
if OrgCache.boundaries in caches:
Boundary.fetch_boundaries(org)
if OrgCache.main_polls in caches:
refresh_main_poll.delay(org.pk)
if OrgCache.brick_polls in caches:
refresh_brick_polls.delay(org.pk)
if OrgCache.other_polls in caches:
refresh_other_polls.delay(org.pk)
if OrgCache.flows in caches:
refresh_org_flows.delay(org.pk)
if OrgCache.recent_reporters in caches:
fetch_contacts_task.delay(org.pk)
if OrgCache.all_reporters in caches:
fetch_contacts_task.delay(org.pk, True)
def get_regions_stats(org):
org_contacts_counts = get_org_contacts_counts(org)
boundaries_name = Boundary.get_org_top_level_boundaries_name(org)
boundaries_stats = {k[6:]: v for k, v in org_contacts_counts.items() if len(k) > 7 and k.startswith("state")}
regions_stats = sorted(
[dict(name=boundaries_name[k], count=v) for k, v in boundaries_stats.items() if k and k in boundaries_name],
key=lambda i: i["count"],
reverse=True,
)
return regions_stats
def get_contact_activity_location(cls, org, time_filter):
now = timezone.now()
today = now.date()
year_ago = now - timedelta(days=365)
start = year_ago.replace(day=1).date()
top_boundaries = Boundary.get_org_top_level_boundaries_name(org)
output_data = []
for osm_id, name in top_boundaries.items():
activities = (ContactActivity.objects.filter(
org=org, date__lte=today, date__gte=start,
state=osm_id).values("date").annotate(Count("id")))
series = ContactActivity.get_activity_data(activities, time_filter)
output_data.append(dict(name=name, osm_id=osm_id, data=series))
return output_data
def get_context_data(self, **kwargs):
context = super(PollContextMixin, self).get_context_data(**kwargs)
org = self.request.org
context["org"] = org
context["states"] = sorted(
[dict(id=k, name=v) for k, v in Boundary.get_org_top_level_boundaries_name(org).items()],
key=lambda c: c["name"],
)
main_poll = self.derive_main_poll()
context["latest_poll"] = main_poll
if main_poll:
top_question = main_poll.get_questions().first()
context["top_question"] = top_question
if top_question:
gender_stats = top_question.get_gender_stats()
total_gender = 0
for elt in gender_stats:
total_gender += elt["set"]
gender_label_dict = {str(v.lower()): k.lower() for k, v in GenderSegment.GENDERS.items()}
gender_stats_dict = {
gender_label_dict.get(elt["label"].lower()): dict(
count=elt["set"], percentage=int(round(elt["set"] * 100 / float(total_gender)))
)
for elt in gender_stats
if total_gender
}
context["gender_stats"] = gender_stats_dict
age_stats = top_question.get_age_stats()
total_age = 0
for elt in age_stats:
total_age += elt["set"]
context["age_stats"] = [
dict(name=elt["label"], y=int(round(elt["set"] * 100 / float(total_age))))
for elt in age_stats
if total_age
]
context["locations_stats"] = top_question.get_location_stats()
if not main_poll or not main_poll.get_questions().first():
context["gender_stats"] = org.get_gender_stats()
context["age_stats"] = org.get_age_stats()
polls = Poll.get_public_polls(org=org).order_by("-poll_date").select_related("category")
categories_dict = defaultdict(list)
for poll in polls:
categories_dict[poll.category.name].append(poll)
context["categories"] = sorted(
[dict(name=k, polls=v) for k, v in categories_dict.items()], key=lambda c: c["name"]
)
context["polls"] = polls
context["main_stories"] = Story.objects.filter(org=org, featured=True, is_active=True).order_by("-created_on")
return context
