Пример #1
0
def agg_arrival_for(indicator, entity, year, month):
    if month is not None:
        periods = [MonthPeriod.get_or_create(year, month)]
    else:
        periods = MonthPeriod.all_from(MonthPeriod.get_or_create(year, 1),
                                       MonthPeriod.get_or_create(year, 12))

    expected_entities = expected_entities_for(indicator, entity)

    qs = DataRecord.objects.filter(indicator=indicator,
                                   period__in=periods) \
                           .filter(entity__in=expected_entities)

    nb_expected_reports = len(expected_entities)
    nb_arrived_reports = qs.count()
    nb_prompt_reports = qs.filter(arrival_status=DataRecord.ARRIVED_ON_TIME) \
                          .count()

    return {
        'nb_expected_reports': nb_expected_reports,
        'nb_arrived_reports': nb_arrived_reports,
        'nb_prompt_reports': nb_prompt_reports,
        'completeness': nb_arrived_reports / nb_expected_reports,
        'promptness': nb_prompt_reports / nb_expected_reports
    }
Пример #2
0
 def year_data_for(self, entity, year, month=None):
     if month is not None:
         periods = [MonthPeriod.get_or_create(year, month)]
     else:
         periods = MonthPeriod.all_from(MonthPeriod.get_or_create(year, 1),
                                        MonthPeriod.get_or_create(year, 12))
     return self.data_for(entity, periods)
Пример #3
0
def agg_arrival_for(indicator, entity, year, month):
    if month is not None:
        periods = [MonthPeriod.get_or_create(year, month)]
    else:
        periods = MonthPeriod.all_from(MonthPeriod.get_or_create(year, 1),
                                       MonthPeriod.get_or_create(year, 12))

    expected_entities = expected_entities_for(indicator, entity)

    qs = DataRecord.objects.filter(indicator=indicator,
                                   period__in=periods) \
                           .filter(entity__in=expected_entities)

    nb_expected_reports = len(expected_entities)
    nb_arrived_reports = qs.count()
    nb_prompt_reports = qs.filter(arrival_status=DataRecord.ARRIVED_ON_TIME) \
                          .count()

    return {
        'nb_expected_reports': nb_expected_reports,
        'nb_arrived_reports': nb_arrived_reports,
        'nb_prompt_reports': nb_prompt_reports,
        'completeness': nb_arrived_reports / nb_expected_reports,
        'promptness': nb_prompt_reports / nb_expected_reports
    }
Пример #4
0
 def year_data_for(self, entity, year, month=None):
     if month is not None:
         periods = [MonthPeriod.get_or_create(year, month)]
     else:
         periods = MonthPeriod.all_from(
             MonthPeriod.get_or_create(year, 1),
             MonthPeriod.get_or_create(year, 12))
     return self.data_for(entity, periods)
Пример #5
0
def process_period_filter(request, period_str=None, name='period'):
    if period_str:
        period = MonthPeriod.get_or_none(period_str)
        if period is None:
            raise Http404(request,
                          _("Unable to match period with `{period}`")
                          .format(period=period_str))
    else:
        period = MonthPeriod.current().previous()

    all_periods = MonthPeriod.all_till_now()
    return {
        'periods': sorted([p.to_tuple() for p in all_periods],
                          reverse=True),
        'all_periods': all_periods,
        name: period,
    }
Пример #6
0
def dashboard(request, entity_uuid=None, indicator_slug=None,
              period_str=None, *args, **kwargs):
    context = {'page': 'dashboard'}

    # entity
    context.update(process_entity_filter(request, entity_uuid))
    root = context['entity'] if context['entity'] else Entity.get_root()

    context.update(process_period_filter(request, period_str, 'period'))
    if context['period'] is None:
        context['period'] = MonthPeriod.current().previous()

    all_indicators = Indicator.get_all_sorted()  # Indicator.get_all_routine()
    indicator = Indicator.get_or_none(indicator_slug)

    context.update({
        'root': root,
        'completeness': OrderedDict([
            (child, get_cached_data('completeness',
                                    dps=child, period=context['period'],
                                    indicator=indicator))
            for child in root.get_children()
        ]),
        'indicators': all_indicators,
        'indicator': indicator,
        'lineage': [Entity.PROVINCE]
    })

    # totals
    context.update({
        'mean_completeness': numpy.mean(
            [e['completeness'] for e in context['completeness'].values()]),
        'mean_promptness': numpy.mean(
            [e['promptness'] for e in context['completeness'].values()]),
    })

    # evolution of pw_anc_receiving_sp3
    pwsp3 = get_timed_records(Indicator.get_by_number(59),
                              root, context['all_periods'])
    perioda = context['all_periods'][0]
    periodb = context['all_periods'][-1]

    context.update({
        'sp3_title': "{num} : {name} entre {pa} et {pb}"
                     .format(num=pwsp3['indicator'].number,
                             name=pwsp3['indicator'].name,
                             pa=perioda.strid,
                             pb=periodb.strid),
        'sp3_fname': "palu-evol-sp3-_{pa}_{pb}"
                     .format(pa=perioda.strid, pb=periodb.strid),
        'sp3_categories': [p[1].name for p in pwsp3['periods']],
        'sp3_series': [{'name': pwsp3['indicator'].name,
                        'data': pwsp3['points']}
                       ],
    })

    return render(request, kwargs.get('template_name', 'dashboard.html'),
                  context)
Пример #7
0
def png_map_for(request,
                perioda_str,
                periodb_str,
                entity_name,
                indicator_number,
                with_title=True,
                with_legend=True):

    entity = Entity.get_by_short_name(entity_name)
    if entity is None:
        raise Http404(
            _("Unknown entity with name `{u}`").format(u=entity_name))

    if perioda_str is None and periodb_str is None \
            and indicator_number is None:
        periods = None
        indicator = None
        with_title = False
        fname = "initial.png"
    else:
        with_title = True
        perioda = process_period_filter(request, perioda_str,
                                        'perioda').get('perioda')
        periodb = process_period_filter(request, periodb_str,
                                        'periodb').get('periodb')
        periods = MonthPeriod.all_from(perioda, periodb)
        if not len(periods):
            raise Http404(
                _("Unknown period interval `{pa}` `{pb}`").format(
                    pa=perioda_str, pb=periodb_str))

        indicator = Indicator.get_by_number(indicator_number)
        if indicator is None:
            raise Http404(
                _("Unknown indicator `{s}`").format(s=indicator_number))

        fname = fname_for(entity, periods, indicator)

    fpath = os.path.join('png_map', fname)
    abspath = os.path.join(settings.EXPORT_REPOSITORY, fpath)

    if not os.path.exists(abspath):
        try:
            gen_map_for(entity,
                        periods,
                        indicator,
                        save_as=abspath,
                        with_title=with_title,
                        with_index=with_title)
        except IOError:
            logger.error("Missing map png folder in exports.")
            raise

    # return redirect('export', fpath=fpath)
    return serve_exported_files(request, fpath=fpath)
Пример #8
0
def map(request, *args, **kwargs):
    context = {'page': 'map'}
    drc = Entity.get_root()

    context.update({
        'root': drc,
        'periods': MonthPeriod.all_till_now(),
        'dps_list': drc.get_children()
    })

    return render(request, kwargs.get('template_name', 'map.html'), context)
Пример #9
0
def map(request, *args, **kwargs):
    context = {'page': 'map'}
    drc = Entity.get_root()

    context.update({
        'root': drc,
        'periods': MonthPeriod.all_till_now(),
        'dps_list': drc.get_children()
    })

    return render(request, kwargs.get('template_name', 'map.html'), context)
Пример #10
0
Файл: api.py Проект: yeleman/dmd
def png_map_for(request, perioda_str, periodb_str,
                entity_name, indicator_number,
                with_title=True, with_legend=True):

    entity = Entity.get_by_short_name(entity_name)
    if entity is None:
        raise Http404(_("Unknown entity with name `{u}`")
                      .format(u=entity_name))

    if perioda_str is None and periodb_str is None \
            and indicator_number is None:
        periods = None
        indicator = None
        with_title = False
        fname = "initial.png"
    else:
        with_title = True
        perioda = process_period_filter(
            request, perioda_str, 'perioda').get('perioda')
        periodb = process_period_filter(
            request, periodb_str, 'periodb').get('periodb')
        periods = MonthPeriod.all_from(perioda, periodb)
        if not len(periods):
            raise Http404(_("Unknown period interval `{pa}` `{pb}`")
                          .format(pa=perioda_str, pb=periodb_str))

        indicator = Indicator.get_by_number(indicator_number)
        if indicator is None:
            raise Http404(_("Unknown indicator `{s}`")
                          .format(s=indicator_number))

        fname = fname_for(entity, periods, indicator)

    fpath = os.path.join('png_map', fname)
    abspath = os.path.join(settings.EXPORT_REPOSITORY, fpath)

    if not os.path.exists(abspath):
        try:
            gen_map_for(entity, periods, indicator,
                        save_as=abspath,
                        with_title=with_title,
                        with_index=with_title)
        except IOError:
            logger.error("Missing map png folder in exports.")
            raise

    # return redirect('export', fpath=fpath)
    return serve_exported_files(request, fpath=fpath)
Пример #11
0
def validation_tallysheet_download(request, period_str=None, *args, **kwargs):
    context = {}
    dps = request.user.partner.validation_location
    context.update(process_period_filter(request, period_str))
    period = context.get('period') or MonthPeriod.current().previous()

    file_name = validation_tallysheet_fname_for(dps, period)
    file_content = generate_validation_tally_for(dps, period).getvalue()

    response = HttpResponse(file_content,
                            content_type='application/'
                                         'vnd.openxmlformats-officedocument'
                                         '.spreadsheetml.sheet')
    response['Content-Disposition'] = 'attachment; filename="%s"' % file_name
    response['Content-Length'] = len(file_content)

    return response
Пример #12
0
def validation_tallysheet_download(request, period_str=None, *args, **kwargs):
    context = {}
    dps = request.user.partner.validation_location
    context.update(process_period_filter(request, period_str))
    period = context.get('period') or MonthPeriod.current().previous()

    file_name = validation_tallysheet_fname_for(dps, period)
    file_content = generate_validation_tally_for(dps, period).getvalue()

    response = HttpResponse(file_content,
                            content_type='application/'
                            'vnd.openxmlformats-officedocument'
                            '.spreadsheetml.sheet')
    response['Content-Disposition'] = 'attachment; filename="%s"' % file_name
    response['Content-Length'] = len(file_content)

    return response
Пример #13
0
def view(request,
         entity_uuid=None,
         perioda_str=None,
         periodb_str=None,
         indicator_slug=None,
         **kwargs):
    context = {'page': 'analysis_section1'}

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    all_indicators = Indicator.objects.all()
    if indicator_slug:
        qs = all_indicators.filter(slug=indicator_slug)
        indicator = qs.get()
    else:
        qs = indicator = None

    context.update({
        'section':
        text_type(SECTION_ID),
        'section_name':
        SECTION_NAME,
        'elements':
        build_context(periods=periods, entity=context['entity'], qs=qs),
        'indicators':
        all_indicators,
        'indicator':
        indicator,
    })

    # absolute URI for links
    context.update({'baseurl': request.build_absolute_uri()})

    return render(request, kwargs.get('template_name',
                                      'analysis_section1.html'), context)
Пример #14
0
def analysis(request,
             section_id,
             entity_uuid=None,
             perioda_str=None,
             periodb_str=None,
             *args,
             **kwargs):
    ''' Generic view for simple sections using an entity and periods '''

    context = {'page': 'analysis_section{}'.format(section_id)}

    if section_id not in SECTIONS:
        raise Http404(_("Unknown section ID `{sid}`").format(sid=section_id))

    section = import_path('dmd.analysis.section{}'.format(section_id))

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    context.update({
        'section':
        section_id,
        'section_name':
        SECTIONS.get(section_id),
        'elements':
        section.build_context(periods=periods, entity=context['entity'])
    })

    # absolute URI for links
    context.update({'baseurl': request.build_absolute_uri()})

    return render(
        request,
        kwargs.get('template_name',
                   'analysis_section{}.html'.format(section_id)), context)
Пример #15
0
def view(request, entity_uuid=None, perioda_str=None, periodb_str=None,
         indicator_slug=None, **kwargs):
    context = {'page': 'analysis_section1'}

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    all_indicators = Indicator.objects.all()
    if indicator_slug:
        qs = all_indicators.filter(slug=indicator_slug)
        indicator = qs.get()
    else:
        qs = indicator = None

    context.update({
        'section': text_type(SECTION_ID),
        'section_name': SECTION_NAME,
        'elements': build_context(periods=periods,
                                  entity=context['entity'],
                                  qs=qs),
        'indicators': all_indicators,
        'indicator': indicator,
    })

    # absolute URI for links
    context.update({'baseurl': request.build_absolute_uri()})

    return render(request,
                  kwargs.get('template_name',
                             'analysis_section1.html'),
                  context)
Пример #16
0
def analysis(request, section_id,
             entity_uuid=None, perioda_str=None, periodb_str=None,
             *args, **kwargs):
    ''' Generic view for simple sections using an entity and periods '''

    context = {'page': 'analysis_section{}'.format(section_id)}

    if section_id not in SECTIONS:
        raise Http404(_("Unknown section ID `{sid}`").format(sid=section_id))

    section = import_path('dmd.analysis.section{}'.format(section_id))

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    context.update({
        'section': section_id,
        'section_name': SECTIONS.get(section_id),
        'elements': section.build_context(periods=periods,
                                          entity=context['entity'])
    })

    # absolute URI for links
    context.update({'baseurl': request.build_absolute_uri()})

    return render(request,
                  kwargs.get('template_name',
                             'analysis_section{}.html'.format(section_id)),
                  context)
Пример #17
0
    def handle(self, *args, **options):

        # make sure we're at project root
        chdir_dmd()

        # options parsing
        self.debug = options.get('debug')
        update = options.get('update')
        period = MonthPeriod.get_or_none(options.get('period'))
        if period is None:
            logger.error(
                "Unable to match an actual period from `{}`".format(period))

        if options.get('previous', False):
            periods = []
            p = period
            while p > MonthPeriod.objects.all().first():
                periods.append(p)
                if len(periods) >= NB_PREVIOUS_PERIODS:
                    break
                p = p.previous()
        else:
            periods = [period]

        upath = '/analytics.json'

        indicators = {
            i.slug: (i.dhis_numerator_id, i.dhis_denominator_id)
            for i in Indicator.get_all_dhis()
        }
        dhis_ids = list(
            set([v[0] for v in indicators.values()] +
                [v[1] for v in indicators.values()]))

        drc = Entity.get_root()
        params = {
            'dimension': [
                'dx:{}'.format(";".join(dhis_ids)),
                'pe:{}'.format(";".join([pe.dhis_strid for pe in periods]))
            ],
            'filter':
            'ou:{}'.format(drc.dhis_id),
            'displayProperty':
            'NAME',
            'outputIdScheme':
            'ID',
            'skipRounding':
            True,
        }

        logger.info(drc)
        if update or self.no_record_at(entity=drc, period=period):
            self.handle_record(get_dhis(path=upath, params=params),
                               entity=drc,
                               periods=periods)

        for dps in drc.get_children():
            logger.info(dps)

            if not update and not self.no_record_at(entity=dps, period=period):
                continue

            dparams = copy.copy(params)
            dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)})
            self.handle_record(get_dhis(path=upath, params=dparams),
                               entity=dps,
                               periods=periods)

            # don't look for ZS if no data at DPS
            if self.no_record_at(entity=dps, period=period):
                continue

            for zs in dps.get_children():
                logger.info(zs)

                if not update and not self.no_record_at(entity=zs,
                                                        period=period):
                    continue

                zparams = copy.copy(params)
                zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)})
                self.handle_record(get_dhis(path=upath, params=zparams),
                                   entity=zs,
                                   periods=periods)

                # don't look for ZS if no data at DPS
                if self.no_record_at(entity=zs, period=period):
                    continue

                for aire in zs.get_children():
                    logger.info(aire)

                    if not update and not self.no_record_at(entity=aire,
                                                            period=period):
                        continue

                    aparams = copy.copy(params)
                    aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)})
                    self.handle_record(get_dhis(path=upath, params=aparams),
                                       entity=aire,
                                       periods=periods)
Пример #18
0
def dashboard(request,
              entity_uuid=None,
              indicator_slug=None,
              period_str=None,
              *args,
              **kwargs):
    context = {'page': 'dashboard'}

    # entity
    context.update(process_entity_filter(request, entity_uuid))
    root = context['entity'] if context['entity'] else Entity.get_root()

    context.update(process_period_filter(request, period_str, 'period'))
    if context['period'] is None:
        context['period'] = MonthPeriod.current().previous()

    all_indicators = Indicator.get_all_sorted()  # Indicator.get_all_routine()
    indicator = Indicator.get_or_none(indicator_slug)

    context.update({
        'root':
        root,
        'completeness':
        OrderedDict([(child,
                      get_cached_data('completeness',
                                      dps=child,
                                      period=context['period'],
                                      indicator=indicator))
                     for child in root.get_children()]),
        'indicators':
        all_indicators,
        'indicator':
        indicator,
        'lineage': [Entity.PROVINCE]
    })

    # totals
    context.update({
        'mean_completeness':
        numpy.mean(
            [e['completeness'] for e in context['completeness'].values()]),
        'mean_promptness':
        numpy.mean([e['promptness']
                    for e in context['completeness'].values()]),
    })

    # evolution of pw_anc_receiving_sp3
    pwsp3 = get_timed_records(Indicator.get_by_number(59), root,
                              context['all_periods'])
    perioda = context['all_periods'][0]
    periodb = context['all_periods'][-1]

    context.update({
        'sp3_title':
        "{num} : {name} entre {pa} et {pb}".format(
            num=pwsp3['indicator'].number,
            name=pwsp3['indicator'].name,
            pa=perioda.strid,
            pb=periodb.strid),
        'sp3_fname':
        "palu-evol-sp3-_{pa}_{pb}".format(pa=perioda.strid, pb=periodb.strid),
        'sp3_categories': [p[1].name for p in pwsp3['periods']],
        'sp3_series': [{
            'name': pwsp3['indicator'].name,
            'data': pwsp3['points']
        }],
    })

    return render(request, kwargs.get('template_name', 'dashboard.html'),
                  context)
Пример #19
0
def view(request, entity_uuid=None, perioda_str=None, periodb_str=None,
         indicator_slug=None, **kwargs):
    context = {'page': 'analysis_section2'}

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    # context.update(process_period_filter(request, period_str, 'period'))
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    def cached_data_list(entity, periods, indicator):
        return [get_cached_data('section2-arrivals',
                                entity=entity,
                                period=period,
                                indicator=indicator)
                for period in periods]

    context.update({
        'section': text_type(SECTION_ID),
        'section_name': SECTION_NAME,
        'arrivals': OrderedDict(
            [(indicator, cached_data_list(context['entity'],
                                          periods, indicator))
             for indicator in Indicator.get_all_routine()])
    })

    # evolution graph
    cp = {
        'periods': [period.to_tuple() for period in periods],
        'points': [get_cached_data('section2-points',
                                   entity=context['entity'], period=period)
                   for period in periods]
    }
    perioda = periods[0]
    periodb = periods[-1]
    context.update({
        'cp_title': "Évolution de la complétude à {name} entre {pa} et {pb}"
                    .format(name=context['entity'].short_name,
                            pa=perioda.strid,
                            pb=periodb.strid),
        'cp_fname': "completeness-_{pa}_{pb}"
                    .format(pa=perioda.strid, pb=periodb.strid),
        'cp_categories': [p[1].name for p in cp['periods']],
        'cp_series': [{'name': "Complétude", 'data': cp['points']}]
    })

    # absolute URI for links
    context.update({
        'baseurl': request.build_absolute_uri(),
        'lineage': [Entity.PROVINCE]})

    return render(request,
                  kwargs.get('template_name',
                             'analysis_section2.html'),
                  context)
Пример #20
0
    def handle(self, *args, **options):

        # make sure we're at project root
        chdir_dmd()

        logger.info("Updating cache for dashboard completeness...")

        root = Entity.get_root()
        periods = MonthPeriod.all_till_now()
        all_dps = root.get_children()
        all_entities = list(all_dps) + [root]
        indicators = Indicator.objects.all()
        all_indicators = list(indicators) + [None]
        nb_items = len(periods) * len(all_dps) * len(all_indicators)

        nb_ran = 0
        for period in periods:
            # logger.debug("{}".format(period))
            for dps in all_dps:
                # logger.debug("== {}".format(dps))
                for indicator in all_indicators:
                    nb_ran += 1
                    # logger.debug("==== {}".format(indicator))

                    params = {
                        'dps': dps,
                        'period': period,
                        'indicator': indicator
                    }

                    # existing cache 4months+ old are not regenerated
                    if period <= periods[-4]:
                        if cache_exists_for('completeness', **params):
                            # logger.info("***** Skipping existing.")
                            continue

                    update_cached_data('completeness', **params)

                    sys.stdout.write("{}/{} - {}%\r"
                                     .format(nb_ran, nb_items,
                                             int(nb_ran / nb_items * 100)))
                    sys.stdout.flush()

        logger.info("Updating cache for section2/arrivals...")

        nb_items = len(periods) * len(all_dps) * len(indicators)
        nb_ran = 0
        for period in periods:
            for entity in all_entities:
                for indicator in indicators:
                    params = {
                        'entity': entity,
                        'period': period,
                        'indicator': indicator
                    }
                    if period <= periods[-4]:
                        if cache_exists_for('section2-arrivals', **params):
                            continue

                        update_cached_data('section2-arrivals',
                                           entity=entity,
                                           period=period,
                                           indicator=indicator)

                        sys.stdout.write("{}/{} - {}%\r"
                                         .format(nb_ran, nb_items,
                                                 int(nb_ran / nb_items * 100)))
                        sys.stdout.flush()

        logger.info("Updating cache for section2/points")

        nb_items = len(periods) * len(all_dps)
        nb_ran = 0
        for period in periods:
            for entity in all_entities:
                if period <= periods[-4]:
                    if cache_exists_for('section2-points', **params):
                        continue

                    update_cached_data('section2-points',
                                       entity=entity,
                                       period=period)

                    sys.stdout.write("{}/{} - {}%\r"
                                     .format(nb_ran, nb_items,
                                             int(nb_ran / nb_items * 100)))
                    sys.stdout.flush()

        logger.info("done.")
Пример #21
0
def view(request,
         entity_uuid=None,
         perioda_str=None,
         periodb_str=None,
         indicator_slug=None,
         **kwargs):
    context = {'page': 'analysis_section2'}

    # handling entity
    context.update(process_entity_filter(request, entity_uuid))

    # handling periods
    # context.update(process_period_filter(request, period_str, 'period'))
    context.update(process_period_filter(request, perioda_str, 'perioda'))
    context.update(process_period_filter(request, periodb_str, 'periodb'))
    if context['perioda'] > context['periodb']:
        context['perioda'], context['periodb'] = \
            context['periodb'], context['perioda']
    periods = MonthPeriod.all_from(context['perioda'], context['periodb'])
    context.update({'selected_periods': periods})

    def cached_data_list(entity, periods, indicator):
        return [
            get_cached_data('section2-arrivals',
                            entity=entity,
                            period=period,
                            indicator=indicator) for period in periods
        ]

    context.update({
        'section':
        text_type(SECTION_ID),
        'section_name':
        SECTION_NAME,
        'arrivals':
        OrderedDict([(indicator,
                      cached_data_list(context['entity'], periods, indicator))
                     for indicator in Indicator.get_all_routine()])
    })

    # evolution graph
    cp = {
        'periods': [period.to_tuple() for period in periods],
        'points': [
            get_cached_data('section2-points',
                            entity=context['entity'],
                            period=period) for period in periods
        ]
    }
    perioda = periods[0]
    periodb = periods[-1]
    context.update({
        'cp_title':
        "Évolution de la complétude à {name} entre {pa} et {pb}".format(
            name=context['entity'].short_name,
            pa=perioda.strid,
            pb=periodb.strid),
        'cp_fname':
        "completeness-_{pa}_{pb}".format(pa=perioda.strid, pb=periodb.strid),
        'cp_categories': [p[1].name for p in cp['periods']],
        'cp_series': [{
            'name': "Complétude",
            'data': cp['points']
        }]
    })

    # absolute URI for links
    context.update({
        'baseurl': request.build_absolute_uri(),
        'lineage': [Entity.PROVINCE]
    })

    return render(request, kwargs.get('template_name',
                                      'analysis_section2.html'), context)
Пример #22
0
def validation(request, template_name='validation.html'):

    context = {'page': 'validation'}

    # recent periods for tally suggestion
    recent_periods = [MonthPeriod.current().previous()]
    for __ in range(2):
        recent_periods.append(recent_periods[-1].previous())
    context.update({'recent_periods': recent_periods})

    now = timezone.now()
    rdc = Entity.get_root()
    validation_location = request.user.partner.validation_location

    records = DataRecord.objects \
        .filter(validation_status=DataRecord.NOT_VALIDATED)

    if validation_location != rdc:
        other_dps = list(rdc.get_children())
        other_dps.remove(validation_location)
        records = records.exclude(entity__in=other_dps) \
                         .exclude(entity__parent__in=other_dps)
    # set order
    records = records.order_by('-created_on')

    nb_total = records.count()
    records = records[:MAX_RESULTS]

    if request.method == 'POST':
        form = ValidationForm(request.POST, records=records)
        if form.is_valid():
            counts = {
                'untouched': 0,
                'updated': 0,
                'validated': 0,
                'rejected': 0
            }

            for dr in records:
                status = form.cleaned_data.get('status-{}'.format(dr.id))
                if status == form.WAIT:
                    counts['untouched'] += 1
                    continue

                if status == 'edit':
                    numerator = form.cleaned_data.get('numerator-{}'
                                                      .format(dr.id))
                    denominator = form.cleaned_data.get('denominator-{}'
                                                        .format(dr.id))

                    dr.numerator = numerator
                    dr.denominator = denominator
                    dr.record_update(request.user.partner)

                    counts['updated'] += 1

                    status = form.VALIDATED

                if status in (form.VALIDATED, form.REJECTED):
                    if status == form.VALIDATED:
                        counts['validated'] += 1
                    else:
                        counts['rejected'] += 1

                    dr.record_validation(
                        status=status,
                        on=now,
                        by=request.user.partner)
            messages.info(request, _(
                "Thank You for your validations. {w} data were left "
                "untouched, {u} were updated, {v} validated "
                "and {r} rejected.").format(
                    w=counts['untouched'],
                    u=counts['updated'],
                    v=counts['validated'],
                    r=counts['rejected']))

            return redirect('validation')
        else:
            # django form validation errors
            pass
    else:
        form = ValidationForm(records=records)

    context.update({
        'form': form,
        'records': records,
        'nb_total': nb_total
    })

    return render(request, template_name, context)
Пример #23
0
    def handle(self, *args, **options):

        # make sure we're at project root
        chdir_dmd()

        logger.info("Updating cache for dashboard completeness...")

        root = Entity.get_root()
        periods = MonthPeriod.all_till_now()
        all_dps = root.get_children()
        all_entities = list(all_dps) + [root]
        indicators = Indicator.objects.all()
        all_indicators = list(indicators) + [None]
        nb_items = len(periods) * len(all_dps) * len(all_indicators)

        nb_ran = 0
        for period in periods:
            # logger.debug("{}".format(period))
            for dps in all_dps:
                # logger.debug("== {}".format(dps))
                for indicator in all_indicators:
                    nb_ran += 1
                    # logger.debug("==== {}".format(indicator))

                    params = {
                        'dps': dps,
                        'period': period,
                        'indicator': indicator
                    }

                    # existing cache 4months+ old are not regenerated
                    if period <= periods[-4]:
                        if cache_exists_for('completeness', **params):
                            # logger.info("***** Skipping existing.")
                            continue

                    update_cached_data('completeness', **params)

                    sys.stdout.write("{}/{} - {}%\r".format(
                        nb_ran, nb_items, int(nb_ran / nb_items * 100)))
                    sys.stdout.flush()

        logger.info("Updating cache for section2/arrivals...")

        nb_items = len(periods) * len(all_dps) * len(indicators)
        nb_ran = 0
        for period in periods:
            for entity in all_entities:
                for indicator in indicators:
                    params = {
                        'entity': entity,
                        'period': period,
                        'indicator': indicator
                    }
                    if period <= periods[-4]:
                        if cache_exists_for('section2-arrivals', **params):
                            continue

                        update_cached_data('section2-arrivals',
                                           entity=entity,
                                           period=period,
                                           indicator=indicator)

                        sys.stdout.write("{}/{} - {}%\r".format(
                            nb_ran, nb_items, int(nb_ran / nb_items * 100)))
                        sys.stdout.flush()

        logger.info("Updating cache for section2/points")

        nb_items = len(periods) * len(all_dps)
        nb_ran = 0
        for period in periods:
            for entity in all_entities:
                if period <= periods[-4]:
                    if cache_exists_for('section2-points', **params):
                        continue

                    update_cached_data('section2-points',
                                       entity=entity,
                                       period=period)

                    sys.stdout.write("{}/{} - {}%\r".format(
                        nb_ran, nb_items, int(nb_ran / nb_items * 100)))
                    sys.stdout.flush()

        logger.info("done.")
Пример #24
0
def read_xls(filepath, partner):

    if not partner.can_upload:
        raise UploadPermissionDenied(_("{user} is not allowed to submit data")
                                     .format(user=partner))

    try:
        wb = load_workbook(filepath, data_only=True)
    except InvalidFileException:
        raise IncorrectExcelFile(_("Not a proper XLSX Template."))

    ws = wb.active

    nb_rows = len(ws.rows)
    cd = lambda row, column: ws.cell(row=row, column=column).value

    # data holder
    data = {
        'errors': [],
    }

    # record now's time to compare with delays
    submited_on = timezone.now()

    def add_error(row, column=None, indicator=None, error=None, text=None):
        data['errors'].append({
            'row': row,
            'column': column,
            'indicator': Indicator.get_or_none(indicator.slug)
            if indicator else None,
            'slug': error,
            'text': text,
        })

    # retrieve and store default year/month
    default_year_addr = "=$C$4"
    default_month_addr = "=$D$4"
    try:
        default_year = int(float(cd(4, 3)))
    except:
        default_year = None

    try:
        default_month = int(float(cd(4, 4)))
    except:
        default_month = None

    for row in range(5, nb_rows + 1):

        row_has_data = sum([1 for x in ws.rows[row - 1][4:] if x.value])
        if not row_has_data:
            continue

        rdc = Entity.get_root()

        dps = Entity.find_by_stdname(cd(row, 1), parent=rdc)
        if dps is None:
            logger.warning("No DPS for row #{}".format(row))
            continue  # no DPS, no data

        zs = Entity.find_by_stdname(cd(row, 2), parent=dps)
        if zs is None:
            if cd(row, 2).lower().strip() != "-":
                logger.warning("No ZS for row #{}".format(row))
                continue  # no ZS, no data
            else:
                entity = dps
        else:
            entity = zs

        # check upload location authorization
        if partner.upload_location not in entity.get_ancestors():
            add_error(row, error='permission_denied',
                      text=_("You can't submit data for "
                             "that location ({entity})")
                      .format(entity=entity))
            continue

        # retrieve period
        year_str = cd(row, 3)
        if year_str == default_year_addr:
            year = default_year
        else:
            try:
                year = int(float(year_str))
            except:
                year = None

        month_str = cd(row, 4)
        if month_str == default_month_addr:
            month = default_month
        else:
            try:
                month = int(float(month_str))
            except:
                month = None

        if year is None or month is None:
            logger.warning("No year or month for row #{}".format(row))
            add_error(row, error='incorrect_period',
                      text=_("Missing year or month at row {row}")
                      .format(row))
            continue

        try:
            period = MonthPeriod.get_or_create(year, month)
        except ValueError as e:
            logger.warning("Unable to retrieve period: {}".format(e))
            add_error(row, error='incorrect_period',
                      text=_("Unable to retrieve period for {y}/{m}")
                      .format(y=year, m=month))
            continue

        for idx, cell in enumerate(ws.rows[2][4:]):
            if idx % 3 != 0:
                continue  # skip empty merged cols

            column = letter_to_column(cell.column)

            try:
                number = cell.value.split('-')[0].strip()
            except:
                # the header doesn't respect the format
                # better fail everything
                raise IncorrectExcelFile(_("Not a proper XLSX Template."))

            num = cd(row, column)
            denom = cd(row, column + 1)

            # skip if missing numerator
            if num is None:
                # logger.debug("No numerator for indic #{}".format(number))
                continue

            indicator = Indicator.get_by_number(number)
            # not an expected number
            if indicator is None:
                logger.warning("No indicator found at col #{}".format(column))
                add_error(row, column=cell.column,
                          error='incorrect_indicator',
                          text=_("Unable to match an Indicator at col {col}")
                                .format(col=cell.column))
                continue

            # ensure level of data submitted depending on type
            if indicator.collection_type == indicator.SURVEY:
                # data must be DPS
                if not entity.is_dps:
                    logger.warning("Data for Survey Indic on non-DPS #{}"
                                   .format(cell.coordinate))
                    add_error(row, column=cell.column,
                              error='incorect_level',
                              text=_("Survey indicator require DPS data"))
                    continue

            elif indicator.collection_type == indicator.ROUTINE:
                # data must be ZS
                if not entity.is_zs:
                    logger.warning("Data for Routine Indic on non-ZS #{}"
                                   .format(cell.coordinate))
                    add_error(row, column=cell.column,
                              error='incorect_level',
                              text=_("Routine indicator require ZS data"))
                    continue

            # check submission period for that Indicator
            if not indicator.can_submit_on(on=submited_on, period=period):
                logger.warning("{on} is not a valid submission time "
                               "for {ind} {period}"
                               .format(on=submited_on, ind=indicator,
                                       period=period))
                add_error(row, column=cell.column,
                          indicator=indicator,
                          error='outside_submussion_delay',
                          text=_("{on} is outside submission period "
                                 "for Indicator #{ind} at {period}")
                          .format(on=submited_on.strftime('%d-%m-%Y'),
                                  ind=indicator.number,
                                  period=period))
                continue

            if not indicator.is_number and denom is None:
                logger.warning("No denominator for indic #{}".format(number))
                add_error(row, column=cell.column,
                          error='missing_denominator',
                          text=_("Missing denominator on "
                                 "non-number Indicator"))
                continue
            elif indicator.is_number:
                denom = 1

            try:
                num = float(num)
                denom = float(denom)
            except:
                add_error(row, column=cell.column,
                          error='incorrect_value',
                          indicator=indicator,
                          text=_("Incorrect value for numerator "
                                 "or denominator `{num} / {denom}`")
                                .format(num=num, denom=denom))
                continue

            data.update({data_ident_for(indicator, period, entity): {
                'slug': indicator.slug,
                'period': period,
                'entity': entity,
                'numerator': num,
                'denominator': denom}})

    return data
Пример #25
0
def validation(request, template_name='validation.html'):

    context = {'page': 'validation'}

    # recent periods for tally suggestion
    recent_periods = [MonthPeriod.current().previous()]
    for __ in range(2):
        recent_periods.append(recent_periods[-1].previous())
    context.update({'recent_periods': recent_periods})

    now = timezone.now()
    rdc = Entity.get_root()
    validation_location = request.user.partner.validation_location

    records = DataRecord.objects \
        .filter(validation_status=DataRecord.NOT_VALIDATED)

    if validation_location != rdc:
        other_dps = list(rdc.get_children())
        other_dps.remove(validation_location)
        records = records.exclude(entity__in=other_dps) \
                         .exclude(entity__parent__in=other_dps)
    # set order
    records = records.order_by('-created_on')

    nb_total = records.count()
    records = records[:MAX_RESULTS]

    if request.method == 'POST':
        form = ValidationForm(request.POST, records=records)
        if form.is_valid():
            counts = {
                'untouched': 0,
                'updated': 0,
                'validated': 0,
                'rejected': 0
            }

            for dr in records:
                status = form.cleaned_data.get('status-{}'.format(dr.id))
                if status == form.WAIT:
                    counts['untouched'] += 1
                    continue

                if status == 'edit':
                    numerator = form.cleaned_data.get('numerator-{}'.format(
                        dr.id))
                    denominator = form.cleaned_data.get(
                        'denominator-{}'.format(dr.id))

                    dr.numerator = numerator
                    dr.denominator = denominator
                    dr.record_update(request.user.partner)

                    counts['updated'] += 1

                    status = form.VALIDATED

                if status in (form.VALIDATED, form.REJECTED):
                    if status == form.VALIDATED:
                        counts['validated'] += 1
                    else:
                        counts['rejected'] += 1

                    dr.record_validation(status=status,
                                         on=now,
                                         by=request.user.partner)
            messages.info(
                request,
                _("Thank You for your validations. {w} data were left "
                  "untouched, {u} were updated, {v} validated "
                  "and {r} rejected.").format(w=counts['untouched'],
                                              u=counts['updated'],
                                              v=counts['validated'],
                                              r=counts['rejected']))

            return redirect('validation')
        else:
            # django form validation errors
            pass
    else:
        form = ValidationForm(records=records)

    context.update({'form': form, 'records': records, 'nb_total': nb_total})

    return render(request, template_name, context)
Пример #26
0
    def handle(self, *args, **options):

        # make sure we're at project root
        chdir_dmd()

        # options parsing
        self.debug = options.get('debug')
        update = options.get('update')
        period = MonthPeriod.get_or_none(options.get('period'))
        if period is None:
            logger.error("Unable to match an actual period from `{}`"
                         .format(period))

        if options.get('previous', False):
            periods = []
            p = period
            while p > MonthPeriod.objects.all().first():
                periods.append(p)
                if len(periods) >= NB_PREVIOUS_PERIODS:
                    break
                p = p.previous()
        else:
            periods = [period]

        upath = '/analytics.json'

        indicators = {i.slug: (i.dhis_numerator_id, i.dhis_denominator_id)
                      for i in Indicator.get_all_dhis()}
        dhis_ids = list(set([v[0] for v in indicators.values()] +
                            [v[1] for v in indicators.values()]))

        drc = Entity.get_root()
        params = {
            'dimension': ['dx:{}'.format(";".join(dhis_ids)),
                          'pe:{}'.format(
                          ";".join([pe.dhis_strid for pe in periods]))],
            'filter': 'ou:{}'.format(drc.dhis_id),
            'displayProperty': 'NAME',
            'outputIdScheme': 'ID',
            'skipRounding': True,
        }

        logger.info(drc)
        if update or self.no_record_at(entity=drc, period=period):
            self.handle_record(get_dhis(path=upath, params=params),
                               entity=drc, periods=periods)

        for dps in drc.get_children():
            logger.info(dps)

            if not update and not self.no_record_at(entity=dps, period=period):
                continue

            dparams = copy.copy(params)
            dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)})
            self.handle_record(get_dhis(path=upath, params=dparams),
                               entity=dps, periods=periods)

            # don't look for ZS if no data at DPS
            if self.no_record_at(entity=dps, period=period):
                continue

            for zs in dps.get_children():
                logger.info(zs)

                if not update and not self.no_record_at(entity=zs,
                                                        period=period):
                    continue

                zparams = copy.copy(params)
                zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)})
                self.handle_record(get_dhis(path=upath, params=zparams),
                                   entity=zs, periods=periods)

                # don't look for ZS if no data at DPS
                if self.no_record_at(entity=zs, period=period):
                    continue

                for aire in zs.get_children():
                    logger.info(aire)

                    if not update and not self.no_record_at(entity=aire,
                                                            period=period):
                        continue

                    aparams = copy.copy(params)
                    aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)})
                    self.handle_record(get_dhis(path=upath, params=aparams),
                                       entity=aire, periods=periods)