def index(): args = request.args.copy() # get variables from query params, or use (hopefully) sensible defaults group = args.pop('group', None) location_type_id = args.pop('locationtype', None) page_title = _('Dashboard') template_name = 'frontend/dashboard.html' event = get_event() if not args.get('checklist_form'): form = forms.find( events=event, form_type='CHECKLIST').order_by('name').first() else: form = forms.get_or_404(pk=args.get('checklist_form')) if form: args.setdefault('checklist_form', unicode(form.id)) queryset = submissions.find( form=form, submission_type='M' ) filter_ = dashboard_filterset()(queryset, data=args) obs_queryset = submissions.find( form=form, submission_type='O' ) obs_filter_ = dashboard_filterset()(obs_queryset, data=args) location = None if args.get('location'): location = locations.get_or_404(pk=args.get('location')) # activate sample filter filter_form = filter_.form queryset = filter_.qs obs_queryset = obs_filter_.qs next_location_type = False if not group: data = get_coverage(queryset) obs_data = get_coverage(obs_queryset) else: page_title = page_title + u' · {}'.format(group) if not location_type_id: location_type = location_types.find( is_administrative=True).order_by('ancestor_count').first() else: location_type = LocationType.objects.get_or_404( pk=location_type_id) # get the requisite location type - the way the aggregation # works, passing in a 'State' location type won't retrieve # data for the level below the 'State' type, it will retrieve # it for the 'State' type. in general, this isn't the behaviour # we want, so we need to find the lower level types and get the # one we want (the first of the children) le_temp = [lt for lt in location_type.children if lt.is_administrative] try: next_location_type = le_temp[0] except IndexError: next_location_type = None data = get_coverage(queryset, group, location_type) obs_data = get_coverage(obs_queryset, group, location_type) # load the page context location_id = args.pop('location', '') context = { 'args': args, 'location_id': location_id, 'next_location': bool(next_location_type), 'data': data, 'obs_data': obs_data, 'filter_form': filter_form, 'page_title': page_title, 'location': location, 'locationtype': getattr(next_location_type, 'id', ''), 'group': group or '' } return render_template( template_name, **context )
def _process_analysis(form_id, location_id=None, tag=None): form = forms.get_or_404(pk=form_id) location = locations.get_or_404(pk=location_id) \ if location_id else locations.root() template_name = '' tags = [] page_title = _(u'%(form)s Analysis', form=form.name) grouped = False display_tag = None filter_class = filters.generate_submission_analysis_filter(form) # set the correct template and fill out the required data if form.form_type == 'CHECKLIST': if tag: template_name = 'process_analysis/checklist_summary_breakdown.html' tags.append(tag) display_tag = tag grouped = True else: template_name = 'process_analysis/checklist_summary.html' tags.extend([ field.name for group in form.groups for field in group.fields if field.analysis_type == 'PROCESS' ]) grouped = False queryset = submissions.find( form=form, submission_type='M', quarantine_status__nin=['A']).filter_in(location) else: grouped = True queryset = submissions.find(form=form).filter_in(location) template_name = 'process_analysis/critical_incident_summary.html' if tag: # a slightly different filter, one prefiltering # on the specified tag display_tag = tag template_name = 'process_analysis/critical_incidents_locations.html' filter_class = \ filters.generate_critical_incident_location_filter(tag) # create data filter filter_set = filter_class(queryset, request.args) # set up template context context = {} context['dataframe'] = filter_set.qs.to_dataframe() context['page_title'] = page_title context['display_tag'] = display_tag context['filter_form'] = filter_set.form context['form'] = form context['location'] = location context['field_groups'] = OrderedDict() context['breadcrumb_data'] = analysis_breadcrumb_data( form, location, display_tag) context['navigation_data'] = analysis_navigation_data( form, location, display_tag) for group in form.groups: process_fields = sorted([ field for field in group.fields if field.analysis_type == 'PROCESS' ], key=lambda x: x.name) context['field_groups'][group.name] = process_fields # processing for incident forms if form.form_type == 'INCIDENT': if display_tag: context['form_field'] = form.get_field_by_tag(display_tag) context['location_types'] = location_types.find(is_political=True) context['incidents'] = filter_set.qs else: incidents_summary = generate_incidents_data( form, filter_set.qs, location, grouped, tags) context['incidents_summary'] = incidents_summary else: process_summary = generate_process_data(form, filter_set.qs, location, grouped=True, tags=tags) context['process_summary'] = process_summary return render_template(template_name, **context)
def _voting_results(form_pk, location_pk=None): form = forms.get_or_404(pk=form_pk, form_type='CHECKLIST', groups__fields__analysis_type='RESULT') if location_pk is None: location = locations.root() else: location = locations.get_or_404(pk=location_pk) template_name = 'result_analysis/results.html' page_title = _(u'%(form_name)s Voting Results', form_name=form.name) filter_class = filters.generate_submission_analysis_filter(form) loc_types = [ lt for lt in location_types.root().children if lt.is_political is True ] location_tree = { lt.name: locations.find(location_type=lt.name).order_by('name') for lt in loc_types } # define the condition for which a submission should be included result_fields = filter( lambda field: field.analysis_type == 'RESULT', [field for group in form.groups for field in group.fields]) result_field_labels = [field.name for field in result_fields] result_field_descriptions = [field.description for field in result_fields] queryset = submissions.find( form=form, submission_type='M', verification_status__ne=FLAG_STATUSES['rejected'][0], quarantine_status__nin=['A', 'R'], ) filter_set = filter_class(queryset, request.args) dataset = filter_set.qs.to_dataframe() registered_voters_field = form.registered_voters_tag or 'registered_voters' if form.invalid_votes_tag: rejected_votes_field = [form.invalid_votes_tag] else: rejected_votes_field = [] if form.blank_votes_tag: blank_votes_field = [form.blank_votes_tag] else: blank_votes_field = [] # compute and store reporting status dataset['reported'] = dataset[result_field_labels + [registered_voters_field]].count( 1) == len(result_field_labels) + 1 dataset['missing'] = dataset[result_field_labels + [registered_voters_field]].count( 1) != len(result_field_labels) + 1 try: overall_summation = dataset.groupby( location.location_type).sum().ix[location.name] reported_subset = dataset[dataset.reported == True] valid_dataframe = reported_subset[reported_subset[ location.location_type] == location.name] valid_summation = reported_subset.fillna(0).groupby( location.location_type).sum().ix[location.name] reporting = overall_summation[['missing', 'reported']] reporting['reported_pct'] = reporting['reported'] / ( reporting['reported'] + reporting['missing']) reporting['missing_pct'] = reporting['missing'] / ( reporting['reported'] + reporting['missing']) data_analyses = {'overall': {}, 'grouped': {}} data_analyses['overall'] = { 'reported_cnt': int(reporting['reported']), 'missing_cnt': int(reporting['missing']), 'reported_pct': reporting['reported_pct'], 'missing_pct': reporting['missing_pct'], 'rv': valid_summation.get(registered_voters_field, 0), 'all_votes': int(valid_summation[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0)), 'turnout': valid_summation[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) / valid_summation.get(registered_voters_field, 0) or pd.np.inf, 'all_valid_votes': int(valid_summation[result_field_labels].sum(axis=0)), 'all_valid_votes_pct': valid_summation[result_field_labels].sum(axis=0) / valid_summation[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0), 'total_rejected': int(valid_summation[rejected_votes_field[0]]) if rejected_votes_field else 0, 'total_rejected_pct': valid_summation[rejected_votes_field[0]] / valid_summation[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) if rejected_votes_field else 0, 'total_blanks': int(valid_summation[blank_votes_field[0]]) if blank_votes_field else 0, 'total_blanks_pct': valid_summation[blank_votes_field[0]] / valid_summation[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) if blank_votes_field else 0 } if form.calculate_moe and current_app.config.get('ENABLE_MOE'): data_analyses['overall']['turnout_moe_95'] = \ _margin_of_error( valid_dataframe, result_field_labels + rejected_votes_field + blank_votes_field, [registered_voters_field]) data_analyses['overall']['turnout_moe_99'] = \ _margin_of_error( valid_dataframe, result_field_labels + rejected_votes_field + blank_votes_field, [registered_voters_field], 2.58) data_analyses['overall']['all_valid_votes_moe_95'] = \ _margin_of_error( valid_dataframe, result_field_labels, result_field_labels + rejected_votes_field + blank_votes_field) data_analyses['overall']['all_valid_votes_moe_99'] = \ _margin_of_error( valid_dataframe, result_field_labels, result_field_labels + rejected_votes_field + blank_votes_field, 2.58) data_analyses['overall']['total_rejected_moe_95'] = \ _margin_of_error( valid_dataframe, rejected_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field) if rejected_votes_field else 0 data_analyses['overall']['total_rejected_moe_99'] = \ _margin_of_error( valid_dataframe, rejected_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field, 2.58) if rejected_votes_field else 0 data_analyses['overall']['total_blanks_moe_95'] = \ _margin_of_error( valid_dataframe, blank_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field) if blank_votes_field else 0 data_analyses['overall']['total_blanks_moe_99'] = \ _margin_of_error( valid_dataframe, blank_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field, 2.58) if blank_votes_field else 0 data_available = True except KeyError: data_analyses = {'overall': {}, 'grouped': {}} data_analyses['overall'] = { 'reported_cnt': 0, 'missing_cnt': 0, 'reported_pct': 0, 'missing_pct': 0, 'rv': 0, 'all_votes': 0, 'turnout': 0, 'all_valid_votes': 0, 'all_valid_votes_pct': 0, 'total_rejected': 0, 'total_rejected_pct': 0, 'turnout_moe_95': 0, 'turnout_moe_99': 0, 'all_valid_votes_moe_95': 0, 'all_valid_votes_moe_99': 0, 'total_rejected_moe_95': 0, 'total_rejected_moe_99': 0, 'total_blanks_moe_95': 0, 'total_blanks_moe_99': 0 } data_available = False for result_field_label in result_field_labels: data_analyses['overall'][u'{}_cnt'.format(result_field_label)] = \ valid_summation.get(result_field_label, 0) if data_available else 0 data_analyses['overall'][u'{}_pct'.format(result_field_label)] = \ valid_summation.get(result_field_label, 0) / float( data_analyses['overall']['all_valid_votes']) \ if data_available else 0 # all_votes? if form.calculate_moe and current_app.config.get('ENABLE_MOE'): data_analyses['overall'][u'{}_moe_95'.format( result_field_label)] = _margin_of_error( valid_dataframe, result_field_label, result_field_labels) \ if data_available else 0 data_analyses['overall'][u'{}_moe_99'.format( result_field_label)] = _margin_of_error( valid_dataframe, result_field_label, result_field_labels, 2.58) \ if data_available else 0 # grouped summaries for location_type in location_tree.keys(): data_analyses['grouped'][location_type] = [] try: grouped_summation = dataset.groupby(location_type).sum() grouped_valid_summation = dataset[ dataset.reported == True].groupby(location_type).sum() for sublocation in location_tree[location_type]: try: _overall = grouped_summation.ix[sublocation.name] _valid = grouped_valid_summation.fillna(0).ix[ sublocation.name] _reported_subset = dataset[dataset.reported == True] _valid_dataframe = _reported_subset[ _reported_subset[location_type] == sublocation.name] _reporting = _overall[['missing', 'reported']] _reporting['reported_pct'] = _reporting['reported'] / ( _reporting['reported'] + _reporting['missing']) _reporting['missing_pct'] = _reporting['missing'] / ( _reporting['reported'] + _reporting['missing']) _sublocation_report = { 'name': sublocation.name, 'location_type': sublocation.location_type, 'reported_cnt': int(_reporting['reported']), 'reported_pct': _reporting['reported_pct'], 'missing_cnt': int(_reporting['missing']), 'missing_pct': _reporting['missing_pct'], 'rv': _valid.get(registered_voters_field, 0), 'all_votes': int(_valid[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0)), 'turnout': _valid[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) / _valid.get(registered_voters_field, 0) or pd.np.inf, 'all_valid_votes': int(_valid[result_field_labels].sum(axis=0)), 'all_valid_votes_pct': _valid[result_field_labels].sum(axis=0) / _valid[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0), 'total_rejected': int(_valid[rejected_votes_field[0]]) if rejected_votes_field else 0, 'total_rejected_pct': _valid[rejected_votes_field[0]] / _valid[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) if rejected_votes_field else 0, 'total_blanks': int(_valid[blank_votes_field[0]]) if blank_votes_field else 0, 'total_blanks_pct': _valid[blank_votes_field[0]] / _valid[result_field_labels + rejected_votes_field + blank_votes_field].sum(axis=0) if blank_votes_field else 0, } if (form.calculate_moe and current_app.config.get('ENABLE_MOE')): _sublocation_report['turnout_moe_95'] = \ _margin_of_error( _valid_dataframe, result_field_labels + rejected_votes_field + blank_votes_field, [registered_voters_field]) _sublocation_report['turnout_moe_99'] = \ _margin_of_error( _valid_dataframe, result_field_labels + rejected_votes_field + blank_votes_field, [registered_voters_field], 2.58) _sublocation_report['all_valid_votes_moe_95'] = \ _margin_of_error( _valid_dataframe, result_field_labels, result_field_labels + rejected_votes_field + blank_votes_field) _sublocation_report['all_valid_votes_moe_99'] = \ _margin_of_error( _valid_dataframe, result_field_labels, result_field_labels + rejected_votes_field + blank_votes_field, 2.58) _sublocation_report['total_rejected_moe_95'] = \ _margin_of_error( _valid_dataframe, rejected_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field) \ if rejected_votes_field else 0 _sublocation_report['total_rejected_moe_99'] = \ _margin_of_error( _valid_dataframe, rejected_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field, 2.58) \ if rejected_votes_field else 0 _sublocation_report['total_blanks_moe_95'] = \ _margin_of_error( _valid_dataframe, blank_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field) \ if blank_votes_field else 0 _sublocation_report['total_blanks_moe_99'] = \ _margin_of_error( _valid_dataframe, blank_votes_field[0], result_field_labels + rejected_votes_field + blank_votes_field, 2.58) \ if blank_votes_field else 0 data_available = True except KeyError: data_available = False _sublocation_report = { 'name': sublocation.name, 'location_type': sublocation.location_type, 'reported_cnt': 0, 'reported_pct': 0, 'missing_cnt': 0, 'missing_pct': 0, 'rv': 0, 'all_votes': 0, 'turnout': 0, 'all_valid_votes': 0, 'all_valid_votes_pct': 0, 'total_rejected': 0, 'total_rejected_pct': 0, 'turnout_moe_95': 0, 'turnout_moe_99': 0, 'all_valid_votes_moe_95': 0, 'all_valid_votes_moe_99': 0, 'total_rejected_moe_95': 0, 'total_rejected_moe_99': 0, 'total_blanks_moe_95': 0, 'total_blanks_moe_99': 0 } for result_field_label in result_field_labels: _sublocation_report[u'{}_cnt'.format( result_field_label)] = _valid.get( result_field_label, 0) if data_available else 0 _sublocation_report[u'{}_pct'.format( result_field_label)] = _valid.get( result_field_label, 0) / float( _sublocation_report['all_valid_votes']) \ if data_available else 0 if (form.calculate_moe and current_app.config.get('ENABLE_MOE')): _sublocation_report[u'{}_moe_95'.format( result_field_label)] = _margin_of_error( _valid_dataframe, result_field_label, result_field_labels) \ if data_available else 0 _sublocation_report[u'{}_moe_99'.format( result_field_label)] = _margin_of_error( _valid_dataframe, result_field_label, result_field_labels, 2.58) if data_available else 0 data_analyses['grouped'][location_type].append( _sublocation_report) except IndexError: pass convergence_dataset = dataset[dataset.reported == True].fillna(0) chart_data = {} # restrict the convergence dataframe to result fields and compute the # cummulative sum if not convergence_dataset.empty: convergence_df = convergence_dataset.sort( 'updated')[['updated'] + result_field_labels] for field in result_field_labels: convergence_df[field] = convergence_df[field].cumsum() # compute vote proportions A / (A + B + C + ...) convergence_df[result_field_labels] = \ convergence_df[result_field_labels].div( convergence_df[result_field_labels].sum(axis=1), axis=0).fillna(0) for component in result_field_labels: chart_data[component] = map( lambda (ts, f): (int(ts.strftime('%s')) * 1000, f * 100), convergence_df.as_matrix(['updated', component])) context = { 'page_title': page_title, 'filter_form': filter_set.form, 'location': location, 'dataframe': dataset, 'form': form, 'result_labels': result_field_labels, 'result_fields': result_fields, 'data_analyses': data_analyses, 'result_descriptions': result_field_descriptions, 'chart_data': chart_data, 'chart_series': result_field_labels, 'location_types': loc_types, 'location_tree': location_tree, 'breadcrumb_data': analysis_breadcrumb_data(form, location, analysis_type='results'), 'navigation_data': analysis_navigation_data(form, location, analysis_type='results') } return render_template(template_name, **context)
def main_dashboard(form_id=None): args = request.args.copy() # get variables from query params, or use (hopefully) sensible defaults group = args.pop('group', None) location_type_id = args.pop('locationtype', None) template_name = 'frontend/dashboard.html' event = get_event() if not form_id: form = forms.find(events=event, form_type='CHECKLIST').order_by('name').first() else: form = forms.get_or_404(pk=form_id, form_type="CHECKLIST") if form is not None: page_title = _(u'Dashboard · %(name)s', name=form.name) else: page_title = _(u'Dashboard') queryset = submissions.find(form=form, submission_type='M') filter_ = dashboard_filterset()(queryset, data=args) obs_queryset = submissions.find(form=form, submission_type='O') obs_filter_ = dashboard_filterset()(obs_queryset, data=args) location = None if args.get('location'): location = locations.get_or_404(pk=args.get('location')) # activate sample filter filter_form = filter_.form queryset = filter_.qs obs_queryset = obs_filter_.qs next_location_type = False if not group: data = get_coverage(queryset) obs_data = get_coverage(obs_queryset) else: page_title = page_title + u' · {}'.format(group) if not location_type_id: location_type = location_types.find( is_administrative=True).order_by('ancestor_count').first() else: location_type = LocationType.objects.get_or_404( pk=location_type_id) # get the requisite location type - the way the aggregation # works, passing in a 'State' location type won't retrieve # data for the level below the 'State' type, it will retrieve # it for the 'State' type. in general, this isn't the behaviour # we want, so we need to find the lower level types and get the # one we want (the first of the children) le_temp = [lt for lt in location_type.children if lt.is_administrative] try: next_location_type = le_temp[0] except IndexError: next_location_type = None data = get_coverage(queryset, group, location_type) obs_data = get_coverage(obs_queryset, group, location_type) # load the page context location_id = args.pop('location', '') context = { 'args': args, 'location_id': location_id, 'next_location': bool(next_location_type), 'data': data, 'obs_data': obs_data, 'filter_form': filter_form, 'page_title': page_title, 'location': location, 'locationtype': getattr(next_location_type, 'id', ''), 'group': group or '', 'form_id': unicode(form.pk) if form else None } return render_template(template_name, **context)
def _process_analysis(form_id, location_id=None, tag=None): form = forms.get_or_404(pk=form_id) location = locations.get_or_404(pk=location_id) \ if location_id else locations.root() template_name = '' tags = [] page_title = _(u'%(form)s Analysis', form=form.name) grouped = False display_tag = None filter_class = filters.generate_submission_analysis_filter(form) # set the correct template and fill out the required data if form.form_type == 'CHECKLIST': if tag: template_name = 'process_analysis/checklist_summary_breakdown.html' tags.append(tag) display_tag = tag grouped = True else: template_name = 'process_analysis/checklist_summary.html' tags.extend([ field.name for group in form.groups for field in group.fields if field.analysis_type == 'PROCESS' ]) grouped = False queryset = submissions.find( form=form, submission_type='M', quarantine_status__nin=['A']).filter_in(location) else: grouped = True queryset = submissions.find(form=form).filter_in(location) template_name = 'process_analysis/critical_incident_summary.html' if tag: # a slightly different filter, one prefiltering # on the specified tag display_tag = tag template_name = 'process_analysis/critical_incidents_locations.html' filter_class = \ filters.generate_critical_incident_location_filter(tag) # create data filter filter_set = filter_class(queryset, request.args) # set up template context context = {} context['dataframe'] = filter_set.qs.to_dataframe() context['page_title'] = page_title context['display_tag'] = display_tag context['filter_form'] = filter_set.form context['form'] = form context['location'] = location context['field_groups'] = OrderedDict() context['breadcrumb_data'] = analysis_breadcrumb_data( form, location, display_tag) context['navigation_data'] = analysis_navigation_data( form, location, display_tag) for group in form.groups: process_fields = sorted([ field for field in group.fields if field.analysis_type == 'PROCESS'], key=lambda x: x.name) context['field_groups'][group.name] = process_fields # processing for incident forms if form.form_type == 'INCIDENT': if display_tag: context['form_field'] = form.get_field_by_tag(display_tag) context['location_types'] = location_types.find( is_political=True) context['incidents'] = filter_set.qs else: incidents_summary = generate_incidents_data( form, filter_set.qs, location, grouped, tags) context['incidents_summary'] = incidents_summary else: process_summary = generate_process_data( form, filter_set.qs, location, grouped=True, tags=tags) context['process_summary'] = process_summary return render_template(template_name, **context)