def handle_record(self, jsdata, entity, periods): logger.info(periods) missing = '0.0' data = {} # loop on rows indic_data = {(indic_id, pid): val for indic_id, pid, val in jsdata['rows']} for period in periods: pid = period.dhis_strid for indicator in Indicator.get_all_dhis(): numerator = indic_data.get((indicator.dhis_numerator_id, pid)) denominator = indic_data.get( (indicator.dhis_denominator_id, pid)) if numerator is None or numerator == missing: logger.error("Missing numerator `{}` for `{}`".format( indicator.dhis_numerator_id, indicator)) continue if not indicator.is_number and (denominator is None or denominator == missing): logger.error("Missing denominator `{}` for `{}`".format( indicator.dhis_denominator_id, indicator)) continue logger.debug(data_ident_for(indicator, period, entity)) data.update({ data_ident_for(indicator, period, entity): { 'slug': indicator.slug, 'period': period, 'entity': entity, 'numerator': numerator, 'denominator': denominator } }) d = DataRecord.batch_create(data, dhisbot, source=DataRecord.DHIS, arrival_status=DataRecord.ARRIVED, auto_validate=True) if self.debug: pp(d) return d
def handle_record(self, jsdata, entity, periods): logger.info(periods) missing = '0.0' data = {} # loop on rows indic_data = {(indic_id, pid): val for indic_id, pid, val in jsdata['rows']} for period in periods: pid = period.dhis_strid for indicator in Indicator.get_all_dhis(): numerator = indic_data.get((indicator.dhis_numerator_id, pid)) denominator = indic_data.get( (indicator.dhis_denominator_id, pid)) if numerator is None or numerator == missing: logger.error("Missing numerator `{}` for `{}`" .format(indicator.dhis_numerator_id, indicator)) continue if not indicator.is_number and (denominator is None or denominator == missing): logger.error("Missing denominator `{}` for `{}`" .format(indicator.dhis_denominator_id, indicator)) continue logger.debug(data_ident_for(indicator, period, entity)) data.update({data_ident_for(indicator, period, entity): { 'slug': indicator.slug, 'period': period, 'entity': entity, 'numerator': numerator, 'denominator': denominator}}) d = DataRecord.batch_create(data, dhisbot, source=DataRecord.DHIS, arrival_status=DataRecord.ARRIVED, auto_validate=True) if self.debug: pp(d) return d
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error( "Unable to match an actual period from `{}`".format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = { i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis() } dhis_ids = list( set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': [ 'dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format(";".join([pe.dhis_strid for pe in periods])) ], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error("Unable to match an actual period from `{}`" .format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = {i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis()} dhis_ids = list(set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': ['dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format( ";".join([pe.dhis_strid for pe in periods]))], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)