def detail_stats(facility_id): data = load_reports() facilities = map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0]) filtered_data = [r for r in data if facility_id is None or r['facility'] == facility_id] for r in filtered_data: r['display_time'] = datetime.strptime(r['timestamp'], '%Y-%m-%dT%H:%M:%S').strftime('%d/%m/%y %H:%M') r['site_name'] = facilities[r['facility']]['name'] LIMIT = 50 def month_detail(data, label): return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True)[:LIMIT], 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)], len)) for k in ( 'satisfied', 'wait_bucket', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'clinic_totals': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } return sorted(map_reduce(filtered_data, lambda r: [((r['month'], r['_month']), r)], month_detail).values(), key=lambda e: e['_month'])
def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if k in r])) for k in COMPLAINT_TYPES), 'by_clinic': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], }
def month_detail(data, label): categories = ['satisfied'] categories.extend(COMPLAINT_TYPES) return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True), 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)] if r.get(k) is not None else [], len)) for k in categories), 'clinic_totals': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], }
def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if r[k]])) for k in ( 'waiting_time', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'by_clinic': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], }
def _fac_cache(type=None): if type: facs = Location.objects.filter(type__slug=type) else: facs = Location.objects.all() by_id = map_reduce(facs, lambda f: [(f.id, f)], lambda v: v[0]) return lambda id: by_id.get(id)
def month_detail(data, label): return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True)[:LIMIT], 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)], len)) for k in ( 'satisfied', 'wait_bucket', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'clinic_totals': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], }
def detail_stats(facility_id, user=None, state=None): data = load_reports(user=user, state=state) def fac_filter(r, facility_id): if facility_id is None: return True elif facility_id == -999: # 'other' sites return r['facility'] is None else: return r['facility'] == facility_id filtered_data = [r for r in data if fac_filter(r, facility_id)] facilities = facilities_by_id() def month_detail(data, label): categories = ['satisfied'] categories.extend(COMPLAINT_TYPES) return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True), 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)] if r.get(k) is not None else [], len)) for k in categories), 'clinic_totals': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } by_month = map_reduce(filtered_data, lambda r: [((r['month'], r['_month']), r)]) stats = [month_detail(by_month.get(month_key, []), month_key) for month_key in u.iter_report_range(filtered_data)] return sorted(stats, key=lambda e: e['_month'])
def get_taggable_contacts(state, user): """ Returns a map of location id to location name and the contacts in that location, for all locationsin the path of the state (or any location, if no state is provided. """ def get_state_users(state): if state is None: criteria = {'location__slug': 'nigeria'} else: criteria = {'location__type__slug': 'state', 'location__slug': state} users = Contact.objects.filter(**criteria).select_related() for u in users: if user.id != u.user.id: yield { 'user_id': u.id, 'username': u.user.username, 'first_name': u.first_name, 'last_name': u.last_name, 'state': state or 'national' } taggables = list(get_state_users(None)) if state: taggables.extend(get_state_users(state)) by_state = map_reduce(taggables, lambda u: [(u['state'], u)], lambda v, k: sorted(v, key=lambda u: (u['last_name'], u['first_name']))) by_state = [{'state': k, 'users': v} for k, v in by_state.iteritems()] by_state.sort(key=lambda e: 'zzzzz' if e['state'] == 'national' else e['state']) return by_state
def main_dashboard_stats(): data = load_reports() facilities = map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0]) def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if r[k]])) for k in ( 'waiting_time', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'by_clinic': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } return sorted(map_reduce(data, lambda r: [((r['month'], r['_month']), r)], month_stats).values(), key=lambda e: e['_month'])
def load_reports(user=None): # TODO: filtering by state PBFReport = get_model('dashboard', 'PBFReport') ReportComment = get_model('dashboard', 'ReportComment') ReportCommentView = get_model('dashboard', 'ReportCommentView') facilities = map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0]) reports = [u.extract_report(r) for r in PBFReport.objects.all().select_related()] comments = map_reduce(ReportComment.objects.filter(pbf_report__isnull=False), lambda c: [(c.pbf_report_id, c)]) wait_buckets = [(2, '<2'), (4, '2-4'), (None, '>4')] views = [] if user: views = ReportCommentView.objects.filter(user=user)\ .values_list('report_comment', flat=True) def _get_json(comment): # Add whether or not the comment has been viewed. json = comment.json() json.update({'viewed': comment.pk in views if user else None}) return json for r in reports: u.anonymize_contact(r) ts = datetime.strptime(r['timestamp'], '%Y-%m-%dT%H:%M:%S') r['thread'] = [_get_json(c) for c in sorted(comments.get(r['id'], []), key=lambda c: c.date)] r['month'] = ts.strftime('%b %Y') r['_month'] = ts.strftime('%Y-%m') r['display_time'] = datetime.strptime(r['timestamp'], '%Y-%m-%dT%H:%M:%S').strftime('%d/%m/%y at %H:%M') r['site_name'] = facilities[r['facility']]['name'] if r['for_this_site'] else r['site_other'] if r['waiting_time'] is not None: for thresh, label in wait_buckets: if thresh is None or r['waiting_time'] < thresh: r['wait_bucket'] = label break else: r['wait_bucket'] = None return reports
def detail_stats(facility_id, user=None): data = load_reports(user=user) facilities = map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0]) def fac_filter(r, facility_id): if facility_id is None: return True elif facility_id == 999: # 'other' sites return r['facility'] is None else: return r['facility'] == facility_id filtered_data = [r for r in data if fac_filter(r, facility_id)] LIMIT = 50 def month_detail(data, label): return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True)[:LIMIT], 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)], len)) for k in ( 'satisfied', 'wait_bucket', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'clinic_totals': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } by_month = map_reduce(filtered_data, lambda r: [((r['month'], r['_month']), r)]) stats = [month_detail(by_month.get(month_key, []), month_key) for month_key in u.iter_report_range(filtered_data)] return sorted(stats, key=lambda e: e['_month'])
def main_dashboard_stats(user=None): data = load_reports(user=user) facilities = map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0]) def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if r[k]])) for k in ( 'waiting_time', 'staff_friendliness', 'price_display', 'drug_availability', 'cleanliness', )), 'by_clinic': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } by_month = map_reduce(data, lambda r: [((r['month'], r['_month']), r)]) stats = [month_stats(by_month.get(month_key, []), month_key) for month_key in u.iter_report_range(data)] return sorted(stats, key=lambda e: e['_month'])
def load_reports(state=None, anonymize=True): _loc = u._fac_cache('fug') def extract_report(r): data = u.extract_report(r) fug_id = data['facility'] fug = _loc(fug_id) data['fug'] = fug.name data['facility'] = fug.parent_id return data reports = [extract_report(r) for r in FadamaReport.objects.all().select_related()] facs = facilities_by_id() reports = [r for r in reports if state is None or state == facs[r['facility']]['state']] # todo: these should probably be loaded on-demand for individual reports comments = map_reduce(ReportComment.objects.all(), lambda c: [(c.report_id, c)]) def _ts(r): return datetime.strptime(r['timestamp'], '%Y-%m-%dT%H:%M:%S') for r in reports: if not anonymize: r['_contact'] = r['contact'] u.anonymize_contact(r) r['month'] = _ts(r).strftime('%b %Y') r['_month'] = _ts(r).strftime('%Y-%m') r['thread'] = [c.json() for c in sorted(comments.get(r['id'], []), key=lambda c: c.date)] r['display_time'] = _ts(r).strftime('%d/%m/%y %H:%M') r['site_name'] = facs[r['facility']]['name'] reports_by_contact = map_reduce((r for r in reports if not r['proxy']), lambda r: [(r['contact'], r)]) for r in reports: r['from_same'] = [k['id'] for k in reports_by_contact.get(r['contact'], []) if k != r and abs(_ts(r) - _ts(k)) <= settings.RECENT_REPORTS_FROM_SAME_PHONE_WINDOW] return reports
def main_dashboard_stats(user_state): data = load_reports(user_state) facilities = facilities_by_id() def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if k in r])) for k in COMPLAINT_TYPES), 'by_clinic': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } return sorted(map_reduce(data, lambda r: [((r['month'], r['_month']), r)], month_stats).values(), key=lambda e: e['_month'])
def main_dashboard_stats(user=None, state=None): data = load_reports(state=state, user=user) facilities = facilities_by_id() def month_stats(data, label): return { 'total': len(data), 'satisfaction': map_reduce(data, lambda r: [(r['satisfied'],)], len), 'by_category': dict((k, len([r for r in data if k in r])) for k in COMPLAINT_TYPES), 'by_clinic': [[facilities.get(k), v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } by_month = map_reduce(data, lambda r: [((r['month'], r['_month']), r)]) stats = [month_stats(by_month.get(month_key, []), month_key) for month_key in u.iter_report_range(data)] return sorted(stats, key=lambda e: e['_month'])
def detail_stats(facility_id, user_state): data = load_reports(user_state) facilities = facilities_by_id() filtered_data = [r for r in data if facility_id is None or r['facility'] == facility_id] def month_detail(data, label): categories = ['satisfied'] categories.extend(COMPLAINT_TYPES) return { 'total': len(data), 'logs': sorted(data, key=lambda r: r['timestamp'], reverse=True), 'stats': dict((k, map_reduce(data, lambda r: [(r[k],)] if r.get(k) is not None else [], len)) for k in categories), 'clinic_totals': [[facilities[k], v] for k, v in map_reduce(data, lambda r: [(r['facility'],)], len).iteritems()], 'month': label[0], '_month': label[1], } return sorted(map_reduce(filtered_data, lambda r: [((r['month'], r['_month']), r)], month_detail).values(), key=lambda e: e['_month'])
def get_taggable_contacts(program, state, user): """ Returns a map of location id to location name and the contacts in that location, for all locationsin the path of the state (or any location, if no state is provided. """ def is_program_user(u): def program_member(program): return u.has_perm('dashboard.%s_view' % program) all_programs = ('pbf', 'fadama') # check that user is ONLY a member of the program at hand -- this filters out # supervisory accounts like worldbank (member of all programs) return program_member(program) and all(not program_member(p) for p in all_programs if p != program) def get_state_users(state): if state is None: criteria = {'location__slug': 'nigeria'} else: criteria = {'location__type__slug': 'state', 'location__slug': state} users = Contact.objects.filter(**criteria).select_related() for u in users: if is_program_user(u.user) and user.id != u.user.id: yield { 'user_id': u.id, 'username': u.user.username, 'first_name': u.first_name, 'last_name': u.last_name, 'state': state or 'national' } taggables = list(get_state_users(None)) if state: taggables.extend(get_state_users(state)) by_state = map_reduce(taggables, lambda u: [(u['state'], u)], lambda v, k: sorted(v, key=lambda u: (u['last_name'], u['first_name']))) by_state = [{'state': k, 'users': v} for k, v in by_state.iteritems()] by_state.sort(key=lambda e: 'zzzzz' if e['state'] == 'national' else e['state']) return by_state
def facilities_by_id(): return map_reduce(get_facilities(), lambda e: [(e['id'], e)], lambda v: v[0])
def get_facilities(): facs = u.get_facilities('fca') fugs_by_fca = map_reduce(Location.objects.filter(type__slug='fug'), lambda f: [(f.parent_id, f)]) for f in facs: f['fugs'] = sorted(fug.name for fug in fugs_by_fca.get(f['id'], [])) return facs
def load_reports(user=None, state=None, anonymize=True): FadamaReport = get_model('dashboard', 'FadamaReport') ReportComment = get_model('dashboard', 'ReportComment') ReportCommentView = get_model('dashboard', 'ReportCommentView') facs = facilities_by_id() fugs = u._fac_cache('fug') def extract_report(r): data = u.extract_report(r) loc_id = data['facility'] fug = fugs(loc_id) if fug: data['fug'] = fug.name data['facility'] = fug.parent_id elif facs.get(loc_id): data['fug'] = None data['facility'] = loc_id else: data['fug'] = None data['facility'] = None return data reports = [extract_report(r) for r in FadamaReport.objects.all().select_related()] def filter_state(r, state): if state is None: return True else: reporting_fug = fugs(r['reporting_facility']) if reporting_fug: reporting_fca_id = reporting_fug.parent_id else: reporting_fca_id = r['reporting_facility'] reporting_state = facs[reporting_fca_id]['state'] return state == reporting_state reports = [r for r in reports if filter_state(r, state)] # todo: these should probably be loaded on-demand for individual reports comments = map_reduce(ReportComment.objects.filter(fadama_report__isnull=False), lambda c: [(c.fadama_report_id, c)]) def _ts(r): return datetime.strptime(r['timestamp'], '%Y-%m-%dT%H:%M:%S') views = [] if user: views = ReportCommentView.objects.filter(user=user)\ .values_list('report_comment', flat=True) def _get_json(comment): # Add whether or not the comment has been viewed. json = comment.json() json.update({'viewed': comment.pk in views if user else None}) return json for r in reports: if not anonymize: r['_contact'] = r['contact'] u.anonymize_contact(r) r['month'] = _ts(r).strftime('%b %Y') r['_month'] = _ts(r).strftime('%Y-%m') r['thread'] = [_get_json(c) for c in sorted(comments.get(r['id'], []), key=lambda c: c.date)] r['display_time'] = _ts(r).strftime('%d/%m/%y at %H:%M') r['site_name'] = facs[r['facility']]['name'] if r['for_this_site'] else r['site_other'] reports_by_contact = map_reduce((r for r in reports if not r['proxy']), lambda r: [(r['contact'], r)]) for r in reports: r['from_same'] = [k['id'] for k in reports_by_contact.get(r['contact'], []) if k != r and abs(_ts(r) - _ts(k)) <= settings.RECENT_REPORTS_FROM_SAME_PHONE_WINDOW] return reports