def agg_arrival_for(indicator, entity, year, month): if month is not None: periods = [MonthPeriod.get_or_create(year, month)] else: periods = MonthPeriod.all_from(MonthPeriod.get_or_create(year, 1), MonthPeriod.get_or_create(year, 12)) expected_entities = expected_entities_for(indicator, entity) qs = DataRecord.objects.filter(indicator=indicator, period__in=periods) \ .filter(entity__in=expected_entities) nb_expected_reports = len(expected_entities) nb_arrived_reports = qs.count() nb_prompt_reports = qs.filter(arrival_status=DataRecord.ARRIVED_ON_TIME) \ .count() return { 'nb_expected_reports': nb_expected_reports, 'nb_arrived_reports': nb_arrived_reports, 'nb_prompt_reports': nb_prompt_reports, 'completeness': nb_arrived_reports / nb_expected_reports, 'promptness': nb_prompt_reports / nb_expected_reports }
def year_data_for(self, entity, year, month=None): if month is not None: periods = [MonthPeriod.get_or_create(year, month)] else: periods = MonthPeriod.all_from(MonthPeriod.get_or_create(year, 1), MonthPeriod.get_or_create(year, 12)) return self.data_for(entity, periods)
def year_data_for(self, entity, year, month=None): if month is not None: periods = [MonthPeriod.get_or_create(year, month)] else: periods = MonthPeriod.all_from( MonthPeriod.get_or_create(year, 1), MonthPeriod.get_or_create(year, 12)) return self.data_for(entity, periods)
def process_period_filter(request, period_str=None, name='period'): if period_str: period = MonthPeriod.get_or_none(period_str) if period is None: raise Http404(request, _("Unable to match period with `{period}`") .format(period=period_str)) else: period = MonthPeriod.current().previous() all_periods = MonthPeriod.all_till_now() return { 'periods': sorted([p.to_tuple() for p in all_periods], reverse=True), 'all_periods': all_periods, name: period, }
def dashboard(request, entity_uuid=None, indicator_slug=None, period_str=None, *args, **kwargs): context = {'page': 'dashboard'} # entity context.update(process_entity_filter(request, entity_uuid)) root = context['entity'] if context['entity'] else Entity.get_root() context.update(process_period_filter(request, period_str, 'period')) if context['period'] is None: context['period'] = MonthPeriod.current().previous() all_indicators = Indicator.get_all_sorted() # Indicator.get_all_routine() indicator = Indicator.get_or_none(indicator_slug) context.update({ 'root': root, 'completeness': OrderedDict([ (child, get_cached_data('completeness', dps=child, period=context['period'], indicator=indicator)) for child in root.get_children() ]), 'indicators': all_indicators, 'indicator': indicator, 'lineage': [Entity.PROVINCE] }) # totals context.update({ 'mean_completeness': numpy.mean( [e['completeness'] for e in context['completeness'].values()]), 'mean_promptness': numpy.mean( [e['promptness'] for e in context['completeness'].values()]), }) # evolution of pw_anc_receiving_sp3 pwsp3 = get_timed_records(Indicator.get_by_number(59), root, context['all_periods']) perioda = context['all_periods'][0] periodb = context['all_periods'][-1] context.update({ 'sp3_title': "{num} : {name} entre {pa} et {pb}" .format(num=pwsp3['indicator'].number, name=pwsp3['indicator'].name, pa=perioda.strid, pb=periodb.strid), 'sp3_fname': "palu-evol-sp3-_{pa}_{pb}" .format(pa=perioda.strid, pb=periodb.strid), 'sp3_categories': [p[1].name for p in pwsp3['periods']], 'sp3_series': [{'name': pwsp3['indicator'].name, 'data': pwsp3['points']} ], }) return render(request, kwargs.get('template_name', 'dashboard.html'), context)
def png_map_for(request, perioda_str, periodb_str, entity_name, indicator_number, with_title=True, with_legend=True): entity = Entity.get_by_short_name(entity_name) if entity is None: raise Http404( _("Unknown entity with name `{u}`").format(u=entity_name)) if perioda_str is None and periodb_str is None \ and indicator_number is None: periods = None indicator = None with_title = False fname = "initial.png" else: with_title = True perioda = process_period_filter(request, perioda_str, 'perioda').get('perioda') periodb = process_period_filter(request, periodb_str, 'periodb').get('periodb') periods = MonthPeriod.all_from(perioda, periodb) if not len(periods): raise Http404( _("Unknown period interval `{pa}` `{pb}`").format( pa=perioda_str, pb=periodb_str)) indicator = Indicator.get_by_number(indicator_number) if indicator is None: raise Http404( _("Unknown indicator `{s}`").format(s=indicator_number)) fname = fname_for(entity, periods, indicator) fpath = os.path.join('png_map', fname) abspath = os.path.join(settings.EXPORT_REPOSITORY, fpath) if not os.path.exists(abspath): try: gen_map_for(entity, periods, indicator, save_as=abspath, with_title=with_title, with_index=with_title) except IOError: logger.error("Missing map png folder in exports.") raise # return redirect('export', fpath=fpath) return serve_exported_files(request, fpath=fpath)
def map(request, *args, **kwargs): context = {'page': 'map'} drc = Entity.get_root() context.update({ 'root': drc, 'periods': MonthPeriod.all_till_now(), 'dps_list': drc.get_children() }) return render(request, kwargs.get('template_name', 'map.html'), context)
def png_map_for(request, perioda_str, periodb_str, entity_name, indicator_number, with_title=True, with_legend=True): entity = Entity.get_by_short_name(entity_name) if entity is None: raise Http404(_("Unknown entity with name `{u}`") .format(u=entity_name)) if perioda_str is None and periodb_str is None \ and indicator_number is None: periods = None indicator = None with_title = False fname = "initial.png" else: with_title = True perioda = process_period_filter( request, perioda_str, 'perioda').get('perioda') periodb = process_period_filter( request, periodb_str, 'periodb').get('periodb') periods = MonthPeriod.all_from(perioda, periodb) if not len(periods): raise Http404(_("Unknown period interval `{pa}` `{pb}`") .format(pa=perioda_str, pb=periodb_str)) indicator = Indicator.get_by_number(indicator_number) if indicator is None: raise Http404(_("Unknown indicator `{s}`") .format(s=indicator_number)) fname = fname_for(entity, periods, indicator) fpath = os.path.join('png_map', fname) abspath = os.path.join(settings.EXPORT_REPOSITORY, fpath) if not os.path.exists(abspath): try: gen_map_for(entity, periods, indicator, save_as=abspath, with_title=with_title, with_index=with_title) except IOError: logger.error("Missing map png folder in exports.") raise # return redirect('export', fpath=fpath) return serve_exported_files(request, fpath=fpath)
def validation_tallysheet_download(request, period_str=None, *args, **kwargs): context = {} dps = request.user.partner.validation_location context.update(process_period_filter(request, period_str)) period = context.get('period') or MonthPeriod.current().previous() file_name = validation_tallysheet_fname_for(dps, period) file_content = generate_validation_tally_for(dps, period).getvalue() response = HttpResponse(file_content, content_type='application/' 'vnd.openxmlformats-officedocument' '.spreadsheetml.sheet') response['Content-Disposition'] = 'attachment; filename="%s"' % file_name response['Content-Length'] = len(file_content) return response
def view(request, entity_uuid=None, perioda_str=None, periodb_str=None, indicator_slug=None, **kwargs): context = {'page': 'analysis_section1'} # handling entity context.update(process_entity_filter(request, entity_uuid)) # handling periods context.update(process_period_filter(request, perioda_str, 'perioda')) context.update(process_period_filter(request, periodb_str, 'periodb')) if context['perioda'] > context['periodb']: context['perioda'], context['periodb'] = \ context['periodb'], context['perioda'] periods = MonthPeriod.all_from(context['perioda'], context['periodb']) context.update({'selected_periods': periods}) all_indicators = Indicator.objects.all() if indicator_slug: qs = all_indicators.filter(slug=indicator_slug) indicator = qs.get() else: qs = indicator = None context.update({ 'section': text_type(SECTION_ID), 'section_name': SECTION_NAME, 'elements': build_context(periods=periods, entity=context['entity'], qs=qs), 'indicators': all_indicators, 'indicator': indicator, }) # absolute URI for links context.update({'baseurl': request.build_absolute_uri()}) return render(request, kwargs.get('template_name', 'analysis_section1.html'), context)
def analysis(request, section_id, entity_uuid=None, perioda_str=None, periodb_str=None, *args, **kwargs): ''' Generic view for simple sections using an entity and periods ''' context = {'page': 'analysis_section{}'.format(section_id)} if section_id not in SECTIONS: raise Http404(_("Unknown section ID `{sid}`").format(sid=section_id)) section = import_path('dmd.analysis.section{}'.format(section_id)) # handling entity context.update(process_entity_filter(request, entity_uuid)) # handling periods context.update(process_period_filter(request, perioda_str, 'perioda')) context.update(process_period_filter(request, periodb_str, 'periodb')) if context['perioda'] > context['periodb']: context['perioda'], context['periodb'] = \ context['periodb'], context['perioda'] periods = MonthPeriod.all_from(context['perioda'], context['periodb']) context.update({'selected_periods': periods}) context.update({ 'section': section_id, 'section_name': SECTIONS.get(section_id), 'elements': section.build_context(periods=periods, entity=context['entity']) }) # absolute URI for links context.update({'baseurl': request.build_absolute_uri()}) return render( request, kwargs.get('template_name', 'analysis_section{}.html'.format(section_id)), context)
def analysis(request, section_id, entity_uuid=None, perioda_str=None, periodb_str=None, *args, **kwargs): ''' Generic view for simple sections using an entity and periods ''' context = {'page': 'analysis_section{}'.format(section_id)} if section_id not in SECTIONS: raise Http404(_("Unknown section ID `{sid}`").format(sid=section_id)) section = import_path('dmd.analysis.section{}'.format(section_id)) # handling entity context.update(process_entity_filter(request, entity_uuid)) # handling periods context.update(process_period_filter(request, perioda_str, 'perioda')) context.update(process_period_filter(request, periodb_str, 'periodb')) if context['perioda'] > context['periodb']: context['perioda'], context['periodb'] = \ context['periodb'], context['perioda'] periods = MonthPeriod.all_from(context['perioda'], context['periodb']) context.update({'selected_periods': periods}) context.update({ 'section': section_id, 'section_name': SECTIONS.get(section_id), 'elements': section.build_context(periods=periods, entity=context['entity']) }) # absolute URI for links context.update({'baseurl': request.build_absolute_uri()}) return render(request, kwargs.get('template_name', 'analysis_section{}.html'.format(section_id)), context)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error( "Unable to match an actual period from `{}`".format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = { i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis() } dhis_ids = list( set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': [ 'dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format(";".join([pe.dhis_strid for pe in periods])) ], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)
def dashboard(request, entity_uuid=None, indicator_slug=None, period_str=None, *args, **kwargs): context = {'page': 'dashboard'} # entity context.update(process_entity_filter(request, entity_uuid)) root = context['entity'] if context['entity'] else Entity.get_root() context.update(process_period_filter(request, period_str, 'period')) if context['period'] is None: context['period'] = MonthPeriod.current().previous() all_indicators = Indicator.get_all_sorted() # Indicator.get_all_routine() indicator = Indicator.get_or_none(indicator_slug) context.update({ 'root': root, 'completeness': OrderedDict([(child, get_cached_data('completeness', dps=child, period=context['period'], indicator=indicator)) for child in root.get_children()]), 'indicators': all_indicators, 'indicator': indicator, 'lineage': [Entity.PROVINCE] }) # totals context.update({ 'mean_completeness': numpy.mean( [e['completeness'] for e in context['completeness'].values()]), 'mean_promptness': numpy.mean([e['promptness'] for e in context['completeness'].values()]), }) # evolution of pw_anc_receiving_sp3 pwsp3 = get_timed_records(Indicator.get_by_number(59), root, context['all_periods']) perioda = context['all_periods'][0] periodb = context['all_periods'][-1] context.update({ 'sp3_title': "{num} : {name} entre {pa} et {pb}".format( num=pwsp3['indicator'].number, name=pwsp3['indicator'].name, pa=perioda.strid, pb=periodb.strid), 'sp3_fname': "palu-evol-sp3-_{pa}_{pb}".format(pa=perioda.strid, pb=periodb.strid), 'sp3_categories': [p[1].name for p in pwsp3['periods']], 'sp3_series': [{ 'name': pwsp3['indicator'].name, 'data': pwsp3['points'] }], }) return render(request, kwargs.get('template_name', 'dashboard.html'), context)
def view(request, entity_uuid=None, perioda_str=None, periodb_str=None, indicator_slug=None, **kwargs): context = {'page': 'analysis_section2'} # handling entity context.update(process_entity_filter(request, entity_uuid)) # handling periods # context.update(process_period_filter(request, period_str, 'period')) context.update(process_period_filter(request, perioda_str, 'perioda')) context.update(process_period_filter(request, periodb_str, 'periodb')) if context['perioda'] > context['periodb']: context['perioda'], context['periodb'] = \ context['periodb'], context['perioda'] periods = MonthPeriod.all_from(context['perioda'], context['periodb']) context.update({'selected_periods': periods}) def cached_data_list(entity, periods, indicator): return [get_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) for period in periods] context.update({ 'section': text_type(SECTION_ID), 'section_name': SECTION_NAME, 'arrivals': OrderedDict( [(indicator, cached_data_list(context['entity'], periods, indicator)) for indicator in Indicator.get_all_routine()]) }) # evolution graph cp = { 'periods': [period.to_tuple() for period in periods], 'points': [get_cached_data('section2-points', entity=context['entity'], period=period) for period in periods] } perioda = periods[0] periodb = periods[-1] context.update({ 'cp_title': "Évolution de la complétude à {name} entre {pa} et {pb}" .format(name=context['entity'].short_name, pa=perioda.strid, pb=periodb.strid), 'cp_fname': "completeness-_{pa}_{pb}" .format(pa=perioda.strid, pb=periodb.strid), 'cp_categories': [p[1].name for p in cp['periods']], 'cp_series': [{'name': "Complétude", 'data': cp['points']}] }) # absolute URI for links context.update({ 'baseurl': request.build_absolute_uri(), 'lineage': [Entity.PROVINCE]}) return render(request, kwargs.get('template_name', 'analysis_section2.html'), context)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Updating cache for dashboard completeness...") root = Entity.get_root() periods = MonthPeriod.all_till_now() all_dps = root.get_children() all_entities = list(all_dps) + [root] indicators = Indicator.objects.all() all_indicators = list(indicators) + [None] nb_items = len(periods) * len(all_dps) * len(all_indicators) nb_ran = 0 for period in periods: # logger.debug("{}".format(period)) for dps in all_dps: # logger.debug("== {}".format(dps)) for indicator in all_indicators: nb_ran += 1 # logger.debug("==== {}".format(indicator)) params = { 'dps': dps, 'period': period, 'indicator': indicator } # existing cache 4months+ old are not regenerated if period <= periods[-4]: if cache_exists_for('completeness', **params): # logger.info("***** Skipping existing.") continue update_cached_data('completeness', **params) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/arrivals...") nb_items = len(periods) * len(all_dps) * len(indicators) nb_ran = 0 for period in periods: for entity in all_entities: for indicator in indicators: params = { 'entity': entity, 'period': period, 'indicator': indicator } if period <= periods[-4]: if cache_exists_for('section2-arrivals', **params): continue update_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/points") nb_items = len(periods) * len(all_dps) nb_ran = 0 for period in periods: for entity in all_entities: if period <= periods[-4]: if cache_exists_for('section2-points', **params): continue update_cached_data('section2-points', entity=entity, period=period) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("done.")
def view(request, entity_uuid=None, perioda_str=None, periodb_str=None, indicator_slug=None, **kwargs): context = {'page': 'analysis_section2'} # handling entity context.update(process_entity_filter(request, entity_uuid)) # handling periods # context.update(process_period_filter(request, period_str, 'period')) context.update(process_period_filter(request, perioda_str, 'perioda')) context.update(process_period_filter(request, periodb_str, 'periodb')) if context['perioda'] > context['periodb']: context['perioda'], context['periodb'] = \ context['periodb'], context['perioda'] periods = MonthPeriod.all_from(context['perioda'], context['periodb']) context.update({'selected_periods': periods}) def cached_data_list(entity, periods, indicator): return [ get_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) for period in periods ] context.update({ 'section': text_type(SECTION_ID), 'section_name': SECTION_NAME, 'arrivals': OrderedDict([(indicator, cached_data_list(context['entity'], periods, indicator)) for indicator in Indicator.get_all_routine()]) }) # evolution graph cp = { 'periods': [period.to_tuple() for period in periods], 'points': [ get_cached_data('section2-points', entity=context['entity'], period=period) for period in periods ] } perioda = periods[0] periodb = periods[-1] context.update({ 'cp_title': "Évolution de la complétude à {name} entre {pa} et {pb}".format( name=context['entity'].short_name, pa=perioda.strid, pb=periodb.strid), 'cp_fname': "completeness-_{pa}_{pb}".format(pa=perioda.strid, pb=periodb.strid), 'cp_categories': [p[1].name for p in cp['periods']], 'cp_series': [{ 'name': "Complétude", 'data': cp['points'] }] }) # absolute URI for links context.update({ 'baseurl': request.build_absolute_uri(), 'lineage': [Entity.PROVINCE] }) return render(request, kwargs.get('template_name', 'analysis_section2.html'), context)
def validation(request, template_name='validation.html'): context = {'page': 'validation'} # recent periods for tally suggestion recent_periods = [MonthPeriod.current().previous()] for __ in range(2): recent_periods.append(recent_periods[-1].previous()) context.update({'recent_periods': recent_periods}) now = timezone.now() rdc = Entity.get_root() validation_location = request.user.partner.validation_location records = DataRecord.objects \ .filter(validation_status=DataRecord.NOT_VALIDATED) if validation_location != rdc: other_dps = list(rdc.get_children()) other_dps.remove(validation_location) records = records.exclude(entity__in=other_dps) \ .exclude(entity__parent__in=other_dps) # set order records = records.order_by('-created_on') nb_total = records.count() records = records[:MAX_RESULTS] if request.method == 'POST': form = ValidationForm(request.POST, records=records) if form.is_valid(): counts = { 'untouched': 0, 'updated': 0, 'validated': 0, 'rejected': 0 } for dr in records: status = form.cleaned_data.get('status-{}'.format(dr.id)) if status == form.WAIT: counts['untouched'] += 1 continue if status == 'edit': numerator = form.cleaned_data.get('numerator-{}' .format(dr.id)) denominator = form.cleaned_data.get('denominator-{}' .format(dr.id)) dr.numerator = numerator dr.denominator = denominator dr.record_update(request.user.partner) counts['updated'] += 1 status = form.VALIDATED if status in (form.VALIDATED, form.REJECTED): if status == form.VALIDATED: counts['validated'] += 1 else: counts['rejected'] += 1 dr.record_validation( status=status, on=now, by=request.user.partner) messages.info(request, _( "Thank You for your validations. {w} data were left " "untouched, {u} were updated, {v} validated " "and {r} rejected.").format( w=counts['untouched'], u=counts['updated'], v=counts['validated'], r=counts['rejected'])) return redirect('validation') else: # django form validation errors pass else: form = ValidationForm(records=records) context.update({ 'form': form, 'records': records, 'nb_total': nb_total }) return render(request, template_name, context)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Updating cache for dashboard completeness...") root = Entity.get_root() periods = MonthPeriod.all_till_now() all_dps = root.get_children() all_entities = list(all_dps) + [root] indicators = Indicator.objects.all() all_indicators = list(indicators) + [None] nb_items = len(periods) * len(all_dps) * len(all_indicators) nb_ran = 0 for period in periods: # logger.debug("{}".format(period)) for dps in all_dps: # logger.debug("== {}".format(dps)) for indicator in all_indicators: nb_ran += 1 # logger.debug("==== {}".format(indicator)) params = { 'dps': dps, 'period': period, 'indicator': indicator } # existing cache 4months+ old are not regenerated if period <= periods[-4]: if cache_exists_for('completeness', **params): # logger.info("***** Skipping existing.") continue update_cached_data('completeness', **params) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/arrivals...") nb_items = len(periods) * len(all_dps) * len(indicators) nb_ran = 0 for period in periods: for entity in all_entities: for indicator in indicators: params = { 'entity': entity, 'period': period, 'indicator': indicator } if period <= periods[-4]: if cache_exists_for('section2-arrivals', **params): continue update_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/points") nb_items = len(periods) * len(all_dps) nb_ran = 0 for period in periods: for entity in all_entities: if period <= periods[-4]: if cache_exists_for('section2-points', **params): continue update_cached_data('section2-points', entity=entity, period=period) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("done.")
def read_xls(filepath, partner): if not partner.can_upload: raise UploadPermissionDenied(_("{user} is not allowed to submit data") .format(user=partner)) try: wb = load_workbook(filepath, data_only=True) except InvalidFileException: raise IncorrectExcelFile(_("Not a proper XLSX Template.")) ws = wb.active nb_rows = len(ws.rows) cd = lambda row, column: ws.cell(row=row, column=column).value # data holder data = { 'errors': [], } # record now's time to compare with delays submited_on = timezone.now() def add_error(row, column=None, indicator=None, error=None, text=None): data['errors'].append({ 'row': row, 'column': column, 'indicator': Indicator.get_or_none(indicator.slug) if indicator else None, 'slug': error, 'text': text, }) # retrieve and store default year/month default_year_addr = "=$C$4" default_month_addr = "=$D$4" try: default_year = int(float(cd(4, 3))) except: default_year = None try: default_month = int(float(cd(4, 4))) except: default_month = None for row in range(5, nb_rows + 1): row_has_data = sum([1 for x in ws.rows[row - 1][4:] if x.value]) if not row_has_data: continue rdc = Entity.get_root() dps = Entity.find_by_stdname(cd(row, 1), parent=rdc) if dps is None: logger.warning("No DPS for row #{}".format(row)) continue # no DPS, no data zs = Entity.find_by_stdname(cd(row, 2), parent=dps) if zs is None: if cd(row, 2).lower().strip() != "-": logger.warning("No ZS for row #{}".format(row)) continue # no ZS, no data else: entity = dps else: entity = zs # check upload location authorization if partner.upload_location not in entity.get_ancestors(): add_error(row, error='permission_denied', text=_("You can't submit data for " "that location ({entity})") .format(entity=entity)) continue # retrieve period year_str = cd(row, 3) if year_str == default_year_addr: year = default_year else: try: year = int(float(year_str)) except: year = None month_str = cd(row, 4) if month_str == default_month_addr: month = default_month else: try: month = int(float(month_str)) except: month = None if year is None or month is None: logger.warning("No year or month for row #{}".format(row)) add_error(row, error='incorrect_period', text=_("Missing year or month at row {row}") .format(row)) continue try: period = MonthPeriod.get_or_create(year, month) except ValueError as e: logger.warning("Unable to retrieve period: {}".format(e)) add_error(row, error='incorrect_period', text=_("Unable to retrieve period for {y}/{m}") .format(y=year, m=month)) continue for idx, cell in enumerate(ws.rows[2][4:]): if idx % 3 != 0: continue # skip empty merged cols column = letter_to_column(cell.column) try: number = cell.value.split('-')[0].strip() except: # the header doesn't respect the format # better fail everything raise IncorrectExcelFile(_("Not a proper XLSX Template.")) num = cd(row, column) denom = cd(row, column + 1) # skip if missing numerator if num is None: # logger.debug("No numerator for indic #{}".format(number)) continue indicator = Indicator.get_by_number(number) # not an expected number if indicator is None: logger.warning("No indicator found at col #{}".format(column)) add_error(row, column=cell.column, error='incorrect_indicator', text=_("Unable to match an Indicator at col {col}") .format(col=cell.column)) continue # ensure level of data submitted depending on type if indicator.collection_type == indicator.SURVEY: # data must be DPS if not entity.is_dps: logger.warning("Data for Survey Indic on non-DPS #{}" .format(cell.coordinate)) add_error(row, column=cell.column, error='incorect_level', text=_("Survey indicator require DPS data")) continue elif indicator.collection_type == indicator.ROUTINE: # data must be ZS if not entity.is_zs: logger.warning("Data for Routine Indic on non-ZS #{}" .format(cell.coordinate)) add_error(row, column=cell.column, error='incorect_level', text=_("Routine indicator require ZS data")) continue # check submission period for that Indicator if not indicator.can_submit_on(on=submited_on, period=period): logger.warning("{on} is not a valid submission time " "for {ind} {period}" .format(on=submited_on, ind=indicator, period=period)) add_error(row, column=cell.column, indicator=indicator, error='outside_submussion_delay', text=_("{on} is outside submission period " "for Indicator #{ind} at {period}") .format(on=submited_on.strftime('%d-%m-%Y'), ind=indicator.number, period=period)) continue if not indicator.is_number and denom is None: logger.warning("No denominator for indic #{}".format(number)) add_error(row, column=cell.column, error='missing_denominator', text=_("Missing denominator on " "non-number Indicator")) continue elif indicator.is_number: denom = 1 try: num = float(num) denom = float(denom) except: add_error(row, column=cell.column, error='incorrect_value', indicator=indicator, text=_("Incorrect value for numerator " "or denominator `{num} / {denom}`") .format(num=num, denom=denom)) continue data.update({data_ident_for(indicator, period, entity): { 'slug': indicator.slug, 'period': period, 'entity': entity, 'numerator': num, 'denominator': denom}}) return data
def validation(request, template_name='validation.html'): context = {'page': 'validation'} # recent periods for tally suggestion recent_periods = [MonthPeriod.current().previous()] for __ in range(2): recent_periods.append(recent_periods[-1].previous()) context.update({'recent_periods': recent_periods}) now = timezone.now() rdc = Entity.get_root() validation_location = request.user.partner.validation_location records = DataRecord.objects \ .filter(validation_status=DataRecord.NOT_VALIDATED) if validation_location != rdc: other_dps = list(rdc.get_children()) other_dps.remove(validation_location) records = records.exclude(entity__in=other_dps) \ .exclude(entity__parent__in=other_dps) # set order records = records.order_by('-created_on') nb_total = records.count() records = records[:MAX_RESULTS] if request.method == 'POST': form = ValidationForm(request.POST, records=records) if form.is_valid(): counts = { 'untouched': 0, 'updated': 0, 'validated': 0, 'rejected': 0 } for dr in records: status = form.cleaned_data.get('status-{}'.format(dr.id)) if status == form.WAIT: counts['untouched'] += 1 continue if status == 'edit': numerator = form.cleaned_data.get('numerator-{}'.format( dr.id)) denominator = form.cleaned_data.get( 'denominator-{}'.format(dr.id)) dr.numerator = numerator dr.denominator = denominator dr.record_update(request.user.partner) counts['updated'] += 1 status = form.VALIDATED if status in (form.VALIDATED, form.REJECTED): if status == form.VALIDATED: counts['validated'] += 1 else: counts['rejected'] += 1 dr.record_validation(status=status, on=now, by=request.user.partner) messages.info( request, _("Thank You for your validations. {w} data were left " "untouched, {u} were updated, {v} validated " "and {r} rejected.").format(w=counts['untouched'], u=counts['updated'], v=counts['validated'], r=counts['rejected'])) return redirect('validation') else: # django form validation errors pass else: form = ValidationForm(records=records) context.update({'form': form, 'records': records, 'nb_total': nb_total}) return render(request, template_name, context)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error("Unable to match an actual period from `{}`" .format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = {i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis()} dhis_ids = list(set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': ['dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format( ";".join([pe.dhis_strid for pe in periods]))], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)