def get_location_response_rate_series(cls, org, time_filter): now = timezone.now() year_ago = now - timedelta(days=365) start = year_ago.replace(day=1) top_boundaries = Boundary.get_org_top_level_boundaries_name(org) output_data = [] for osm_id, name in top_boundaries.items(): boundary_ids = list( Boundary.objects.filter(org=org).filter( Q(osm_id=osm_id) | Q(parent__osm_id=osm_id) | Q(parent__parent__osm_id=osm_id)).values_list("pk", flat=True)) polled_stats = (PollStats.objects.filter( org=org, date__gte=start, location_id__in=boundary_ids).values("date").annotate( Sum("count"))) responded_stats = (PollStats.objects.filter( org=org, date__gte=start, location_id__in=boundary_ids).exclude( category=None).values("date").annotate(Sum("count"))) series = PollStats.get_response_rate_data(polled_stats, responded_stats, time_filter) output_data.append(dict(name=name, osm_id=osm_id, data=series)) return output_data
def get_context_data(self, **kwargs): context = super(PollContextMixin, self).get_context_data(**kwargs) org = self.request.org context["org"] = org context["gender_stats"] = org.get_gender_stats() context["age_stats"] = org.get_age_stats() context["states"] = [dict(id=k, name=v) for k, v in Boundary.get_org_top_level_boundaries_name(org).items()] main_poll = self.derive_main_poll() context["latest_poll"] = main_poll if main_poll: top_question = main_poll.get_questions().first() context["top_question"] = top_question if top_question: gender_stats = top_question.get_gender_stats() total_gender = 0 for elt in gender_stats: total_gender += elt["set"] gender_stats_dict = { elt["label"].lower(): dict( count=elt["set"], percentage=int(round(elt["set"] * 100 / float(total_gender))) ) for elt in gender_stats if total_gender } context["gender_stats"] = gender_stats_dict age_stats = top_question.get_age_stats() total_age = 0 for elt in age_stats: total_age += elt["set"] context["age_stats"] = [ dict(name=elt["label"], y=int(round(elt["set"] * 100 / float(total_age)))) for elt in age_stats if total_age ] context["locations_stats"] = top_question.get_location_stats() context["categories"] = ( Category.objects.filter(org=org, is_active=True) .prefetch_related(Prefetch("polls", queryset=Poll.objects.filter(is_active=True).order_by("-poll_date"))) .order_by("name") ) context["polls"] = Poll.get_public_polls(org=org).order_by("-poll_date") context["related_stories"] = [] if main_poll: related_stories = Story.objects.filter(org=org, is_active=True, category=main_poll.category) related_stories = related_stories.order_by("-featured", "-created_on") context["related_stories"] = related_stories context["main_stories"] = Story.objects.filter(org=org, featured=True, is_active=True).order_by("-created_on") return context
def refresh_caches(org, caches): if OrgCache.boundaries in caches: Boundary.fetch_boundaries(org) if OrgCache.main_polls in caches: refresh_main_poll.delay(org.pk) if OrgCache.brick_polls in caches: refresh_brick_polls.delay(org.pk) if OrgCache.other_polls in caches: refresh_other_polls.delay(org.pk) if OrgCache.flows in caches: refresh_org_flows.delay(org.pk)
def get_sign_up_rate_location(org, time_filter): now = timezone.now() year_ago = now - timedelta(days=365) start = year_ago.replace(day=1) tz = pytz.timezone("UTC") org_contacts_counts = get_org_contacts_counts(org) registered_on_counts = {k[17:]: v for k, v in org_contacts_counts.items() if k.startswith("registered_state")} top_boundaries = Boundary.get_org_top_level_boundaries_name(org) keys = get_last_months(months_num=time_filter) output_data = [] for osm_id, name in top_boundaries.items(): interval_dict = defaultdict(int) for date_key, date_count in registered_on_counts.items(): if date_key.endswith(f":{osm_id.upper()}"): date_key = date_key[:10] else: continue parsed_time = tz.localize(datetime.strptime(date_key, "%Y-%m-%d")).replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) if parsed_time > start: interval_dict[str(parsed_time.date())] += date_count data = dict() for key in keys: data[key] = interval_dict[key] output_data.append(dict(name=name, osm_id=osm_id, data=data)) return output_data
def fetch_contacts_task(org_id=None, fetch_all=False): r = get_redis_connection() key = 'fetch_contacts' lock_timeout = 3600 if org_id: key = 'fetch_contacts:%d' % org_id lock_timeout = 300 if not r.get(key): with r.lock(key, timeout=lock_timeout): active_orgs = Org.objects.filter(is_active=True) if org_id: active_orgs = Org.objects.filter(pk=org_id) for org in active_orgs: start = time.time() last_fetched_key = Contact.CONTACT_LAST_FETCHED_CACHE_KEY % org.id after = cache.get(last_fetched_key, None) if after: after = json_date_to_datetime(after) if fetch_all: after = None try: if after is None: Boundary.fetch_boundaries(org) ContactField.fetch_contact_fields(org) Boundary.get_boundaries(org) ContactField.get_contact_fields(org) Contact.fetch_contacts(org, after=after) print "Task: fetch_contacts for %s took %ss" % (org.name, time.time() - start) except Exception as e: import traceback traceback.print_exc() logger.exception("Error fetching contacts: %s" % str(e))
def get_context_data(self, **kwargs): context = super(UreportersView, self).get_context_data(**kwargs) org = self.request.org context["org"] = org translation.activate(org.language) # remove the first option '' from calender.month_abbr context["months"] = [ six.text_type(_("%s")) % m for m in calendar.month_abbr ][1:] context["states"] = sorted( [ dict(id=k, name=v) for k, v in Boundary.get_org_top_level_boundaries_name(org).items() ], key=lambda c: c["name"], ) context["gender_stats"] = org.get_gender_stats() context["age_stats"] = json.loads(org.get_age_stats()) context["registration_stats"] = org.get_registration_stats() context["occupation_stats"] = org.get_occupation_stats() context["reporters"] = org.get_reporters_count() context["main_stories"] = Story.objects.filter( org=org, featured=True, is_active=True).order_by("-created_on") # global counter context["global_counter"] = get_global_count() context["average_response_rate"] = PollStats.get_average_response_rate( org) context["data_time_filters"] = [ dict(time_filter_number=key, label=str(val)) for key, val in PollStats.DATA_TIME_FILTERS.items() ] backend_options = org.backends.filter(is_active=True).values_list( "slug", flat=True) show_maps = reduce(operator.or_, [ bool(org.get_config("%s.state_label" % option)) for option in backend_options ], False) context["data_segments"] = [ dict(segment_type=key, label=str(val)) for key, val in PollStats.DATA_SEGMENTS.items() if key != "location" or show_maps ] context["data_metrics"] = [ dict(slug=key, title=str(val)) for key, val in PollStats.DATA_METRICS.items() ] return context
def pull_boundaries(self, org): if org.get_config('is_global'): incoming_objects = Boundary.build_global_boundaries() else: client = self._get_client(org, 1) incoming_objects = client.get_boundaries() return sync_local_to_set(org, BoundarySyncer(), incoming_objects)
def pull_boundaries(self, org): if org.get_config("common.is_global"): incoming_objects = Boundary.build_global_boundaries() else: client = self._get_client(org, 2) incoming_objects = client.get_boundaries(geometry=True).all() return sync_local_to_set(org, BoundarySyncer(backend=self.backend), incoming_objects)
def pull_boundaries(self, org): if org.get_config('common.is_global'): incoming_objects = Boundary.build_global_boundaries() else: client = self._get_client(org, 2) incoming_objects = client.get_boundaries(geometry=True).all() return sync_local_to_set(org, BoundarySyncer(backend=self.backend), incoming_objects)
def get_regions_stats(org): org_contacts_counts = get_org_contacts_counts(org) boundaries_name = Boundary.get_org_top_level_boundaries_name(org) boundaries_stats = {k[6:]: v for k, v in org_contacts_counts.iteritems() if len(k) > 7 and k.startswith('state')} regions_stats = sorted([dict(name=boundaries_name[k], count=v) for k, v in boundaries_stats.iteritems() if k and k in boundaries_name], key=lambda i: i['count'], reverse=True) return regions_stats
def refresh_caches(org, caches): if OrgCache.boundaries in caches: Boundary.fetch_boundaries(org) if OrgCache.main_polls in caches: refresh_main_poll.delay(org.pk) if OrgCache.brick_polls in caches: refresh_brick_polls.delay(org.pk) if OrgCache.other_polls in caches: refresh_other_polls.delay(org.pk) if OrgCache.flows in caches: refresh_org_flows.delay(org.pk) if OrgCache.recent_reporters in caches: fetch_contacts_task.delay(org.pk) if OrgCache.all_reporters in caches: fetch_contacts_task.delay(org.pk, True)
def get_regions_stats(org): org_contacts_counts = get_org_contacts_counts(org) boundaries_name = Boundary.get_org_top_level_boundaries_name(org) boundaries_stats = {k[6:]: v for k, v in org_contacts_counts.items() if len(k) > 7 and k.startswith("state")} regions_stats = sorted( [dict(name=boundaries_name[k], count=v) for k, v in boundaries_stats.items() if k and k in boundaries_name], key=lambda i: i["count"], reverse=True, ) return regions_stats
def get_contact_activity_location(cls, org, time_filter): now = timezone.now() today = now.date() year_ago = now - timedelta(days=365) start = year_ago.replace(day=1).date() top_boundaries = Boundary.get_org_top_level_boundaries_name(org) output_data = [] for osm_id, name in top_boundaries.items(): activities = (ContactActivity.objects.filter( org=org, date__lte=today, date__gte=start, state=osm_id).values("date").annotate(Count("id"))) series = ContactActivity.get_activity_data(activities, time_filter) output_data.append(dict(name=name, osm_id=osm_id, data=series)) return output_data
def get_context_data(self, **kwargs): context = super(PollContextMixin, self).get_context_data(**kwargs) org = self.request.org context["org"] = org context["states"] = sorted( [dict(id=k, name=v) for k, v in Boundary.get_org_top_level_boundaries_name(org).items()], key=lambda c: c["name"], ) main_poll = self.derive_main_poll() context["latest_poll"] = main_poll if main_poll: top_question = main_poll.get_questions().first() context["top_question"] = top_question if top_question: gender_stats = top_question.get_gender_stats() total_gender = 0 for elt in gender_stats: total_gender += elt["set"] gender_label_dict = {str(v.lower()): k.lower() for k, v in GenderSegment.GENDERS.items()} gender_stats_dict = { gender_label_dict.get(elt["label"].lower()): dict( count=elt["set"], percentage=int(round(elt["set"] * 100 / float(total_gender))) ) for elt in gender_stats if total_gender } context["gender_stats"] = gender_stats_dict age_stats = top_question.get_age_stats() total_age = 0 for elt in age_stats: total_age += elt["set"] context["age_stats"] = [ dict(name=elt["label"], y=int(round(elt["set"] * 100 / float(total_age)))) for elt in age_stats if total_age ] context["locations_stats"] = top_question.get_location_stats() if not main_poll or not main_poll.get_questions().first(): context["gender_stats"] = org.get_gender_stats() context["age_stats"] = org.get_age_stats() polls = Poll.get_public_polls(org=org).order_by("-poll_date").select_related("category") categories_dict = defaultdict(list) for poll in polls: categories_dict[poll.category.name].append(poll) context["categories"] = sorted( [dict(name=k, polls=v) for k, v in categories_dict.items()], key=lambda c: c["name"] ) context["polls"] = polls context["main_stories"] = Story.objects.filter(org=org, featured=True, is_active=True).order_by("-created_on") return context