def index():
args = request.args.copy()
# get variables from query params, or use (hopefully) sensible defaults
group = args.pop('group', None)
location_type_id = args.pop('locationtype', None)
page_title = _('Dashboard')
template_name = 'frontend/dashboard.html'
event = get_event()
if not args.get('checklist_form'):
form = forms.find(
events=event, form_type='CHECKLIST').order_by('name').first()
else:
form = forms.get_or_404(pk=args.get('checklist_form'))
if form:
args.setdefault('checklist_form', unicode(form.id))
queryset = submissions.find(
form=form,
submission_type='M'
)
filter_ = dashboard_filterset()(queryset, data=args)
obs_queryset = submissions.find(
form=form,
submission_type='O'
)
obs_filter_ = dashboard_filterset()(obs_queryset, data=args)
location = None
if args.get('location'):
location = locations.get_or_404(pk=args.get('location'))
# activate sample filter
filter_form = filter_.form
queryset = filter_.qs
obs_queryset = obs_filter_.qs
next_location_type = False
if not group:
data = get_coverage(queryset)
obs_data = get_coverage(obs_queryset)
else:
page_title = page_title + u' · {}'.format(group)
if not location_type_id:
location_type = location_types.find(
is_administrative=True).order_by('ancestor_count').first()
else:
location_type = LocationType.objects.get_or_404(
pk=location_type_id)
# get the requisite location type - the way the aggregation
# works, passing in a 'State' location type won't retrieve
# data for the level below the 'State' type, it will retrieve
# it for the 'State' type. in general, this isn't the behaviour
# we want, so we need to find the lower level types and get the
# one we want (the first of the children)
le_temp = [lt for lt in location_type.children
if lt.is_administrative]
try:
next_location_type = le_temp[0]
except IndexError:
next_location_type = None
data = get_coverage(queryset, group, location_type)
obs_data = get_coverage(obs_queryset, group, location_type)
# load the page context
location_id = args.pop('location', '')
context = {
'args': args,
'location_id': location_id,
'next_location': bool(next_location_type),
'data': data,
'obs_data': obs_data,
'filter_form': filter_form,
'page_title': page_title,
'location': location,
'locationtype': getattr(next_location_type, 'id', ''),
'group': group or ''
}
return render_template(
template_name,
**context
)
def _process_analysis(form_id, location_id=None, tag=None):
form = forms.get_or_404(pk=form_id)
location = locations.get_or_404(pk=location_id) \
if location_id else locations.root()
template_name = ''
tags = []
page_title = _(u'%(form)s Analysis', form=form.name)
grouped = False
display_tag = None
filter_class = filters.generate_submission_analysis_filter(form)
# set the correct template and fill out the required data
if form.form_type == 'CHECKLIST':
if tag:
template_name = 'process_analysis/checklist_summary_breakdown.html'
tags.append(tag)
display_tag = tag
grouped = True
else:
template_name = 'process_analysis/checklist_summary.html'
tags.extend([
field.name for group in form.groups for field in group.fields
if field.analysis_type == 'PROCESS'
])
grouped = False
queryset = submissions.find(
form=form, submission_type='M',
quarantine_status__nin=['A']).filter_in(location)
else:
grouped = True
queryset = submissions.find(form=form).filter_in(location)
template_name = 'process_analysis/critical_incident_summary.html'
if tag:
# a slightly different filter, one prefiltering
# on the specified tag
display_tag = tag
template_name = 'process_analysis/critical_incidents_locations.html'
filter_class = \
filters.generate_critical_incident_location_filter(tag)
# create data filter
filter_set = filter_class(queryset, request.args)
# set up template context
context = {}
context['dataframe'] = filter_set.qs.to_dataframe()
context['page_title'] = page_title
context['display_tag'] = display_tag
context['filter_form'] = filter_set.form
context['form'] = form
context['location'] = location
context['field_groups'] = OrderedDict()
context['breadcrumb_data'] = analysis_breadcrumb_data(
form, location, display_tag)
context['navigation_data'] = analysis_navigation_data(
form, location, display_tag)
for group in form.groups:
process_fields = sorted([
field for field in group.fields if field.analysis_type == 'PROCESS'
],
key=lambda x: x.name)
context['field_groups'][group.name] = process_fields
# processing for incident forms
if form.form_type == 'INCIDENT':
if display_tag:
context['form_field'] = form.get_field_by_tag(display_tag)
context['location_types'] = location_types.find(is_political=True)
context['incidents'] = filter_set.qs
else:
incidents_summary = generate_incidents_data(
form, filter_set.qs, location, grouped, tags)
context['incidents_summary'] = incidents_summary
else:
process_summary = generate_process_data(form,
filter_set.qs,
location,
grouped=True,
tags=tags)
context['process_summary'] = process_summary
return render_template(template_name, **context)
def _voting_results(form_pk, location_pk=None):
form = forms.get_or_404(pk=form_pk,
form_type='CHECKLIST',
groups__fields__analysis_type='RESULT')
if location_pk is None:
location = locations.root()
else:
location = locations.get_or_404(pk=location_pk)
template_name = 'result_analysis/results.html'
page_title = _(u'%(form_name)s Voting Results', form_name=form.name)
filter_class = filters.generate_submission_analysis_filter(form)
loc_types = [
lt for lt in location_types.root().children if lt.is_political is True
]
location_tree = {
lt.name: locations.find(location_type=lt.name).order_by('name')
for lt in loc_types
}
# define the condition for which a submission should be included
result_fields = filter(
lambda field: field.analysis_type == 'RESULT',
[field for group in form.groups for field in group.fields])
result_field_labels = [field.name for field in result_fields]
result_field_descriptions = [field.description for field in result_fields]
queryset = submissions.find(
form=form,
submission_type='M',
verification_status__ne=FLAG_STATUSES['rejected'][0],
quarantine_status__nin=['A', 'R'],
)
filter_set = filter_class(queryset, request.args)
dataset = filter_set.qs.to_dataframe()
registered_voters_field = form.registered_voters_tag or 'registered_voters'
if form.invalid_votes_tag:
rejected_votes_field = [form.invalid_votes_tag]
else:
rejected_votes_field = []
if form.blank_votes_tag:
blank_votes_field = [form.blank_votes_tag]
else:
blank_votes_field = []
# compute and store reporting status
dataset['reported'] = dataset[result_field_labels +
[registered_voters_field]].count(
1) == len(result_field_labels) + 1
dataset['missing'] = dataset[result_field_labels +
[registered_voters_field]].count(
1) != len(result_field_labels) + 1
try:
overall_summation = dataset.groupby(
location.location_type).sum().ix[location.name]
reported_subset = dataset[dataset.reported == True]
valid_dataframe = reported_subset[reported_subset[
location.location_type] == location.name]
valid_summation = reported_subset.fillna(0).groupby(
location.location_type).sum().ix[location.name]
reporting = overall_summation[['missing', 'reported']]
reporting['reported_pct'] = reporting['reported'] / (
reporting['reported'] + reporting['missing'])
reporting['missing_pct'] = reporting['missing'] / (
reporting['reported'] + reporting['missing'])
data_analyses = {'overall': {}, 'grouped': {}}
data_analyses['overall'] = {
'reported_cnt':
int(reporting['reported']),
'missing_cnt':
int(reporting['missing']),
'reported_pct':
reporting['reported_pct'],
'missing_pct':
reporting['missing_pct'],
'rv':
valid_summation.get(registered_voters_field, 0),
'all_votes':
int(valid_summation[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)),
'turnout':
valid_summation[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0) /
valid_summation.get(registered_voters_field, 0) or pd.np.inf,
'all_valid_votes':
int(valid_summation[result_field_labels].sum(axis=0)),
'all_valid_votes_pct':
valid_summation[result_field_labels].sum(axis=0) /
valid_summation[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0),
'total_rejected':
int(valid_summation[rejected_votes_field[0]])
if rejected_votes_field else 0,
'total_rejected_pct':
valid_summation[rejected_votes_field[0]] /
valid_summation[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)
if rejected_votes_field else 0,
'total_blanks':
int(valid_summation[blank_votes_field[0]])
if blank_votes_field else 0,
'total_blanks_pct':
valid_summation[blank_votes_field[0]] /
valid_summation[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)
if blank_votes_field else 0
}
if form.calculate_moe and current_app.config.get('ENABLE_MOE'):
data_analyses['overall']['turnout_moe_95'] = \
_margin_of_error(
valid_dataframe, result_field_labels +
rejected_votes_field + blank_votes_field,
[registered_voters_field])
data_analyses['overall']['turnout_moe_99'] = \
_margin_of_error(
valid_dataframe, result_field_labels +
rejected_votes_field + blank_votes_field,
[registered_voters_field], 2.58)
data_analyses['overall']['all_valid_votes_moe_95'] = \
_margin_of_error(
valid_dataframe, result_field_labels,
result_field_labels + rejected_votes_field +
blank_votes_field)
data_analyses['overall']['all_valid_votes_moe_99'] = \
_margin_of_error(
valid_dataframe, result_field_labels,
result_field_labels + rejected_votes_field +
blank_votes_field, 2.58)
data_analyses['overall']['total_rejected_moe_95'] = \
_margin_of_error(
valid_dataframe, rejected_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field) if rejected_votes_field else 0
data_analyses['overall']['total_rejected_moe_99'] = \
_margin_of_error(
valid_dataframe, rejected_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field, 2.58) if rejected_votes_field else 0
data_analyses['overall']['total_blanks_moe_95'] = \
_margin_of_error(
valid_dataframe, blank_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field) if blank_votes_field else 0
data_analyses['overall']['total_blanks_moe_99'] = \
_margin_of_error(
valid_dataframe, blank_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field, 2.58) if blank_votes_field else 0
data_available = True
except KeyError:
data_analyses = {'overall': {}, 'grouped': {}}
data_analyses['overall'] = {
'reported_cnt': 0,
'missing_cnt': 0,
'reported_pct': 0,
'missing_pct': 0,
'rv': 0,
'all_votes': 0,
'turnout': 0,
'all_valid_votes': 0,
'all_valid_votes_pct': 0,
'total_rejected': 0,
'total_rejected_pct': 0,
'turnout_moe_95': 0,
'turnout_moe_99': 0,
'all_valid_votes_moe_95': 0,
'all_valid_votes_moe_99': 0,
'total_rejected_moe_95': 0,
'total_rejected_moe_99': 0,
'total_blanks_moe_95': 0,
'total_blanks_moe_99': 0
}
data_available = False
for result_field_label in result_field_labels:
data_analyses['overall'][u'{}_cnt'.format(result_field_label)] = \
valid_summation.get(result_field_label, 0) if data_available else 0
data_analyses['overall'][u'{}_pct'.format(result_field_label)] = \
valid_summation.get(result_field_label, 0) / float(
data_analyses['overall']['all_valid_votes']) \
if data_available else 0 # all_votes?
if form.calculate_moe and current_app.config.get('ENABLE_MOE'):
data_analyses['overall'][u'{}_moe_95'.format(
result_field_label)] = _margin_of_error(
valid_dataframe, result_field_label,
result_field_labels) \
if data_available else 0
data_analyses['overall'][u'{}_moe_99'.format(
result_field_label)] = _margin_of_error(
valid_dataframe, result_field_label,
result_field_labels, 2.58) \
if data_available else 0
# grouped summaries
for location_type in location_tree.keys():
data_analyses['grouped'][location_type] = []
try:
grouped_summation = dataset.groupby(location_type).sum()
grouped_valid_summation = dataset[
dataset.reported == True].groupby(location_type).sum()
for sublocation in location_tree[location_type]:
try:
_overall = grouped_summation.ix[sublocation.name]
_valid = grouped_valid_summation.fillna(0).ix[
sublocation.name]
_reported_subset = dataset[dataset.reported == True]
_valid_dataframe = _reported_subset[
_reported_subset[location_type] == sublocation.name]
_reporting = _overall[['missing', 'reported']]
_reporting['reported_pct'] = _reporting['reported'] / (
_reporting['reported'] + _reporting['missing'])
_reporting['missing_pct'] = _reporting['missing'] / (
_reporting['reported'] + _reporting['missing'])
_sublocation_report = {
'name':
sublocation.name,
'location_type':
sublocation.location_type,
'reported_cnt':
int(_reporting['reported']),
'reported_pct':
_reporting['reported_pct'],
'missing_cnt':
int(_reporting['missing']),
'missing_pct':
_reporting['missing_pct'],
'rv':
_valid.get(registered_voters_field, 0),
'all_votes':
int(_valid[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)),
'turnout':
_valid[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0) /
_valid.get(registered_voters_field, 0) or pd.np.inf,
'all_valid_votes':
int(_valid[result_field_labels].sum(axis=0)),
'all_valid_votes_pct':
_valid[result_field_labels].sum(axis=0) /
_valid[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0),
'total_rejected':
int(_valid[rejected_votes_field[0]])
if rejected_votes_field else 0,
'total_rejected_pct':
_valid[rejected_votes_field[0]] /
_valid[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)
if rejected_votes_field else 0,
'total_blanks':
int(_valid[blank_votes_field[0]])
if blank_votes_field else 0,
'total_blanks_pct':
_valid[blank_votes_field[0]] /
_valid[result_field_labels + rejected_votes_field +
blank_votes_field].sum(axis=0)
if blank_votes_field else 0,
}
if (form.calculate_moe
and current_app.config.get('ENABLE_MOE')):
_sublocation_report['turnout_moe_95'] = \
_margin_of_error(
_valid_dataframe, result_field_labels +
rejected_votes_field + blank_votes_field,
[registered_voters_field])
_sublocation_report['turnout_moe_99'] = \
_margin_of_error(
_valid_dataframe, result_field_labels +
rejected_votes_field + blank_votes_field,
[registered_voters_field],
2.58)
_sublocation_report['all_valid_votes_moe_95'] = \
_margin_of_error(
_valid_dataframe, result_field_labels,
result_field_labels + rejected_votes_field +
blank_votes_field)
_sublocation_report['all_valid_votes_moe_99'] = \
_margin_of_error(
_valid_dataframe, result_field_labels,
result_field_labels + rejected_votes_field +
blank_votes_field,
2.58)
_sublocation_report['total_rejected_moe_95'] = \
_margin_of_error(
_valid_dataframe, rejected_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field) \
if rejected_votes_field else 0
_sublocation_report['total_rejected_moe_99'] = \
_margin_of_error(
_valid_dataframe, rejected_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field,
2.58) \
if rejected_votes_field else 0
_sublocation_report['total_blanks_moe_95'] = \
_margin_of_error(
_valid_dataframe, blank_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field) \
if blank_votes_field else 0
_sublocation_report['total_blanks_moe_99'] = \
_margin_of_error(
_valid_dataframe, blank_votes_field[0],
result_field_labels + rejected_votes_field +
blank_votes_field,
2.58) \
if blank_votes_field else 0
data_available = True
except KeyError:
data_available = False
_sublocation_report = {
'name': sublocation.name,
'location_type': sublocation.location_type,
'reported_cnt': 0,
'reported_pct': 0,
'missing_cnt': 0,
'missing_pct': 0,
'rv': 0,
'all_votes': 0,
'turnout': 0,
'all_valid_votes': 0,
'all_valid_votes_pct': 0,
'total_rejected': 0,
'total_rejected_pct': 0,
'turnout_moe_95': 0,
'turnout_moe_99': 0,
'all_valid_votes_moe_95': 0,
'all_valid_votes_moe_99': 0,
'total_rejected_moe_95': 0,
'total_rejected_moe_99': 0,
'total_blanks_moe_95': 0,
'total_blanks_moe_99': 0
}
for result_field_label in result_field_labels:
_sublocation_report[u'{}_cnt'.format(
result_field_label)] = _valid.get(
result_field_label, 0) if data_available else 0
_sublocation_report[u'{}_pct'.format(
result_field_label)] = _valid.get(
result_field_label, 0) / float(
_sublocation_report['all_valid_votes']) \
if data_available else 0
if (form.calculate_moe
and current_app.config.get('ENABLE_MOE')):
_sublocation_report[u'{}_moe_95'.format(
result_field_label)] = _margin_of_error(
_valid_dataframe, result_field_label,
result_field_labels) \
if data_available else 0
_sublocation_report[u'{}_moe_99'.format(
result_field_label)] = _margin_of_error(
_valid_dataframe, result_field_label,
result_field_labels,
2.58) if data_available else 0
data_analyses['grouped'][location_type].append(
_sublocation_report)
except IndexError:
pass
convergence_dataset = dataset[dataset.reported == True].fillna(0)
chart_data = {}
# restrict the convergence dataframe to result fields and compute the
# cummulative sum
if not convergence_dataset.empty:
convergence_df = convergence_dataset.sort(
'updated')[['updated'] + result_field_labels]
for field in result_field_labels:
convergence_df[field] = convergence_df[field].cumsum()
# compute vote proportions A / (A + B + C + ...)
convergence_df[result_field_labels] = \
convergence_df[result_field_labels].div(
convergence_df[result_field_labels].sum(axis=1), axis=0).fillna(0)
for component in result_field_labels:
chart_data[component] = map(
lambda (ts, f): (int(ts.strftime('%s')) * 1000, f * 100),
convergence_df.as_matrix(['updated', component]))
context = {
'page_title':
page_title,
'filter_form':
filter_set.form,
'location':
location,
'dataframe':
dataset,
'form':
form,
'result_labels':
result_field_labels,
'result_fields':
result_fields,
'data_analyses':
data_analyses,
'result_descriptions':
result_field_descriptions,
'chart_data':
chart_data,
'chart_series':
result_field_labels,
'location_types':
loc_types,
'location_tree':
location_tree,
'breadcrumb_data':
analysis_breadcrumb_data(form, location, analysis_type='results'),
'navigation_data':
analysis_navigation_data(form, location, analysis_type='results')
}
return render_template(template_name, **context)
def main_dashboard(form_id=None):
args = request.args.copy()
# get variables from query params, or use (hopefully) sensible defaults
group = args.pop('group', None)
location_type_id = args.pop('locationtype', None)
template_name = 'frontend/dashboard.html'
event = get_event()
if not form_id:
form = forms.find(events=event,
form_type='CHECKLIST').order_by('name').first()
else:
form = forms.get_or_404(pk=form_id, form_type="CHECKLIST")
if form is not None:
page_title = _(u'Dashboard · %(name)s', name=form.name)
else:
page_title = _(u'Dashboard')
queryset = submissions.find(form=form, submission_type='M')
filter_ = dashboard_filterset()(queryset, data=args)
obs_queryset = submissions.find(form=form, submission_type='O')
obs_filter_ = dashboard_filterset()(obs_queryset, data=args)
location = None
if args.get('location'):
location = locations.get_or_404(pk=args.get('location'))
# activate sample filter
filter_form = filter_.form
queryset = filter_.qs
obs_queryset = obs_filter_.qs
next_location_type = False
if not group:
data = get_coverage(queryset)
obs_data = get_coverage(obs_queryset)
else:
page_title = page_title + u' · {}'.format(group)
if not location_type_id:
location_type = location_types.find(
is_administrative=True).order_by('ancestor_count').first()
else:
location_type = LocationType.objects.get_or_404(
pk=location_type_id)
# get the requisite location type - the way the aggregation
# works, passing in a 'State' location type won't retrieve
# data for the level below the 'State' type, it will retrieve
# it for the 'State' type. in general, this isn't the behaviour
# we want, so we need to find the lower level types and get the
# one we want (the first of the children)
le_temp = [lt for lt in location_type.children if lt.is_administrative]
try:
next_location_type = le_temp[0]
except IndexError:
next_location_type = None
data = get_coverage(queryset, group, location_type)
obs_data = get_coverage(obs_queryset, group, location_type)
# load the page context
location_id = args.pop('location', '')
context = {
'args': args,
'location_id': location_id,
'next_location': bool(next_location_type),
'data': data,
'obs_data': obs_data,
'filter_form': filter_form,
'page_title': page_title,
'location': location,
'locationtype': getattr(next_location_type, 'id', ''),
'group': group or '',
'form_id': unicode(form.pk) if form else None
}
return render_template(template_name, **context)
def _process_analysis(form_id, location_id=None, tag=None):
form = forms.get_or_404(pk=form_id)
location = locations.get_or_404(pk=location_id) \
if location_id else locations.root()
template_name = ''
tags = []
page_title = _(u'%(form)s Analysis', form=form.name)
grouped = False
display_tag = None
filter_class = filters.generate_submission_analysis_filter(form)
# set the correct template and fill out the required data
if form.form_type == 'CHECKLIST':
if tag:
template_name = 'process_analysis/checklist_summary_breakdown.html'
tags.append(tag)
display_tag = tag
grouped = True
else:
template_name = 'process_analysis/checklist_summary.html'
tags.extend([
field.name for group in form.groups
for field in group.fields if field.analysis_type == 'PROCESS'
])
grouped = False
queryset = submissions.find(
form=form, submission_type='M',
quarantine_status__nin=['A']).filter_in(location)
else:
grouped = True
queryset = submissions.find(form=form).filter_in(location)
template_name = 'process_analysis/critical_incident_summary.html'
if tag:
# a slightly different filter, one prefiltering
# on the specified tag
display_tag = tag
template_name = 'process_analysis/critical_incidents_locations.html'
filter_class = \
filters.generate_critical_incident_location_filter(tag)
# create data filter
filter_set = filter_class(queryset, request.args)
# set up template context
context = {}
context['dataframe'] = filter_set.qs.to_dataframe()
context['page_title'] = page_title
context['display_tag'] = display_tag
context['filter_form'] = filter_set.form
context['form'] = form
context['location'] = location
context['field_groups'] = OrderedDict()
context['breadcrumb_data'] = analysis_breadcrumb_data(
form, location, display_tag)
context['navigation_data'] = analysis_navigation_data(
form, location, display_tag)
for group in form.groups:
process_fields = sorted([
field for field in group.fields
if field.analysis_type == 'PROCESS'], key=lambda x: x.name)
context['field_groups'][group.name] = process_fields
# processing for incident forms
if form.form_type == 'INCIDENT':
if display_tag:
context['form_field'] = form.get_field_by_tag(display_tag)
context['location_types'] = location_types.find(
is_political=True)
context['incidents'] = filter_set.qs
else:
incidents_summary = generate_incidents_data(
form, filter_set.qs, location, grouped, tags)
context['incidents_summary'] = incidents_summary
else:
process_summary = generate_process_data(
form, filter_set.qs, location, grouped=True, tags=tags)
context['process_summary'] = process_summary
return render_template(template_name, **context)
