def handle(self, *args, **options): # make sure we're at project root chdir_dmd() self.json_folder = options.get('json_folder') logger.info("Importing Entities from JSON files from `{}`".format( self.json_folder)) if not p(self.json_folder).isdir(): logger.error("JSON folder `{}` is not a directory.".format( self.json_folder)) for level in range(5): self.enforced_level = level for fpath in os.listdir(self.json_folder): if not fpath.endswith('.json'): continue # import entity eid = fpath[:-5] self.import_entity(eid) self.enforced_level = None
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() self.json_folder = options.get('json_folder') logger.info("Importing Entities from JSON files from `{}`" .format(self.json_folder)) if not p(self.json_folder).isdir(): logger.error("JSON folder `{}` is not a directory." .format(self.json_folder)) for level in range(5): self.enforced_level = level for fpath in os.listdir(self.json_folder): if not fpath.endswith('.json'): continue # import entity eid = fpath[:-5] self.import_entity(eid) self.enforced_level = None
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # django config of the DB dbconf = settings.DATABASES.get('default', {}) if dbconf.get('ENGINE').endswith('mysql'): # dump the MySQL DB to a file dump_name = "{name}.sql".format(name=dbconf.get('NAME')) dump_path = os.path.join(settings.BACKUPS_REPOSITORY, dump_name) cmd = [ 'mysqldump', '-h', dbconf.get('HOST'), '-u', dbconf.get('USER'), '-p{passwd}'.format(passwd=dbconf.get('PASSWORD')), '-r', dump_path, dbconf.get('NAME') ] # dump database subprocess.call(cmd, shell=False) elif dbconf.get('ENGINE').endswith('sqlite3'): # copy sqlite DB to backup dir dump_name = os.path.basename(dbconf.get('NAME')) dump_path = os.path.join(settings.BACKUPS_REPOSITORY, dump_name) shutil.copy2(dbconf.get('NAME'), dump_path) else: logger.error("DB engine `{engine}` not supported".format( engine=dbconf.get('ENGINE'))) # compression is done from backup dir curdir = os.getcwd() os.chdir(settings.BACKUPS_REPOSITORY) ark_name = "{orig}_{date}.7z".format( orig=os.path.basename(dump_path), date=timezone.now().strftime("%Y-%m-%d")) # compress the DB dump subprocess.call(['7z', 'a', ark_name, dump_name]) # delete uncompressed file try: os.unlink(dump_path) except: pass # restore working dir os.chdir(curdir)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # django config of the DB dbconf = settings.DATABASES.get('default', {}) if dbconf.get('ENGINE').endswith('mysql'): # dump the MySQL DB to a file dump_name = "{name}.sql".format(name=dbconf.get('NAME')) dump_path = os.path.join(settings.BACKUPS_REPOSITORY, dump_name) cmd = ['mysqldump', '-h', dbconf.get('HOST'), '-u', dbconf.get('USER'), '-p{passwd}'.format(passwd=dbconf.get('PASSWORD')), '-r', dump_path, dbconf.get('NAME')] # dump database subprocess.call(cmd, shell=False) elif dbconf.get('ENGINE').endswith('sqlite3'): # copy sqlite DB to backup dir dump_name = os.path.basename(dbconf.get('NAME')) dump_path = os.path.join(settings.BACKUPS_REPOSITORY, dump_name) shutil.copy2(dbconf.get('NAME'), dump_path) else: logger.error("DB engine `{engine}` not supported" .format(engine=dbconf.get('ENGINE'))) # compression is done from backup dir curdir = os.getcwd() os.chdir(settings.BACKUPS_REPOSITORY) ark_name = "{orig}_{date}.7z".format( orig=os.path.basename(dump_path), date=timezone.now().strftime("%Y-%m-%d")) # compress the DB dump subprocess.call(['7z', 'a', ark_name, dump_name]) # delete uncompressed file try: os.unlink(dump_path) except: pass # restore working dir os.chdir(curdir)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # daily backups for last 7 days. # Monday backups for last 30 days. # first of each month today = datetime.datetime.today() aweek_ago = today - datetime.timedelta(days=7) amonth_ago = today - datetime.timedelta(days=30) for fname in os.listdir(settings.BACKUPS_REPOSITORY): if fname in ('.', '..', 'README'): continue if not fname.endswith('.7z'): continue fpath = os.path.join(settings.BACKUPS_REPOSITORY, fname) fdate = datetime.datetime( *[int(x) for x in fname.rsplit('.7z', 1)[0] .rsplit('_', 1)[1].split('-')]) # keep all files less than a week old if fdate > aweek_ago: logger.info("Keeping {} - less than a week old".format(fname)) continue # keep every mondays that are less than a month old if fdate.isoweekday() == 1 and fdate > amonth_ago: logger.info("Keeping {} - monday within a month".format(fname)) continue # keep first of all month if fdate.day == 1: logger.info("Keeping {} - first of a month".format(fname)) continue try: logger.info("Removing {}".format(fname)) os.unlink(fpath) except OSError as exp: logger.error("Unable to delete {f}: {exp}" .format(f=fpath, exp=exp))
def handle(self, *args, **kwargs): # make sure we're at project root chdir_dmd() logger.info("Exporting all DataRecord to: {}" .format(settings.ALL_EXPORT_PATH)) qs = get_records() nb_records = qs.count() get_csv_for(qs, save_to=settings.ALL_EXPORT_PATH) logger.info("Exporting all DataRecord (XLS 1sheet/indicator) to {}" .format(settings.ALL_EXPORT_XLSX_PATH)) export_to_spreadsheet(qs, save_to=settings.ALL_EXPORT_XLSX_PATH) Metadata.update('nb_records', nb_records) logger.info("All done")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() dps_name = options.get('dps') if not dps_name: logger.error("Unable to match DPS with name `{}`".format(dps_name)) return 1 rdc = Entity.get_root() dps = Entity.lookup_at(parent=rdc, name=dps_name)[0] if dps is None: logger.error("Unable to match DPS with name `{}`".format(dps_name)) return 1 logger.info("Generating XLS dataentry tool for `{}`".format(dps_name)) generate_dataentry_for(dps, 'dataentry.xlsx')
def handle(self, *args, **kwargs): # make sure we're at project root chdir_dmd() logger.info("Exporting all DataRecord to: {}".format( settings.ALL_EXPORT_PATH)) qs = get_records() nb_records = qs.count() get_csv_for(qs, save_to=settings.ALL_EXPORT_PATH) logger.info( "Exporting all DataRecord (XLS 1sheet/indicator) to {}".format( settings.ALL_EXPORT_XLSX_PATH)) export_to_spreadsheet(qs, save_to=settings.ALL_EXPORT_XLSX_PATH) Metadata.update('nb_records', nb_records) logger.info("All done")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() if not os.path.exists(options.get('file')): logger.error("GeoJSON file does not exit.") return False with open(options.get('file'), 'r') as f: gjson = json.load(f) rdc = Entity.get_root() for feature in gjson['features']: dps_name = feature['properties'].get('NOM_DPS') if dps_name: name = dps_name logger.debug(name) entity = Entity.objects.get(name=name) else: zs_name = feature['properties'].get('NAME') dps_name = feature['properties'].get('DPS') logger.debug("dps: {d} - zs: {z}".format(d=dps_name, z=zs_name)) parent = Entity.find_by_stdname(parent=rdc, std_name=dps_name) logger.debug("\tparent: {p}".format(p=parent)) assert parent is not None entity, children = Entity.lookup_at(parent=parent, name=zs_name.upper()) assert entity is not None logger.info(entity) entity.geometry = json.dumps(feature['geometry']) entity.save() logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() if not os.path.exists(options.get('file')): logger.error("GeoJSON file does not exit.") return False with open(options.get('file'), 'r') as f: gjson = json.load(f) rdc = Entity.get_root() for feature in gjson['features']: dps_name = feature['properties'].get('NOM_DPS') if dps_name: name = dps_name logger.debug(name) entity = Entity.objects.get(name=name) else: zs_name = feature['properties'].get('NAME') dps_name = feature['properties'].get('DPS') logger.debug("dps: {d} - zs: {z}" .format(d=dps_name, z=zs_name)) parent = Entity.find_by_stdname(parent=rdc, std_name=dps_name) logger.debug("\tparent: {p}".format(p=parent)) assert parent is not None entity, children = Entity.lookup_at(parent=parent, name=zs_name.upper()) assert entity is not None logger.info(entity) entity.geometry = json.dumps(feature['geometry']) entity.save() logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Creating latitude and longitude data for Entities") for entity in Entity.objects.all(): if not entity.geometry: logger.debug("Skipping {}, no geometry.".format(entity)) continue feature_geom = shape(entity.geojson["geometry"]) feature_centroid = feature_geom.centroid entity.latitude = feature_centroid.x entity.longitude = feature_centroid.y entity.save() logger.info( "{name}: {lat}, {lng}".format(name=entity.short_name, lat=entity.latitude, lng=entity.longitude) ) logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Creating latitude and longitude data for Entities") for entity in Entity.objects.all(): if not entity.geometry: logger.debug("Skipping {}, no geometry.".format(entity)) continue feature_geom = shape(entity.geojson['geometry']) feature_centroid = feature_geom.centroid entity.latitude = feature_centroid.x entity.longitude = feature_centroid.y entity.save() logger.info("{name}: {lat}, {lng}".format(name=entity.short_name, lat=entity.latitude, lng=entity.longitude)) logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() dps_name = options.get('dps') if not dps_name: logger.error("Unable to match DPS with name `{}`" .format(dps_name)) return 1 rdc = Entity.get_root() dps = Entity.lookup_at(parent=rdc, name=dps_name)[0] if dps is None: logger.error("Unable to match DPS with name `{}`" .format(dps_name)) return 1 logger.info("Generating XLS dataentry tool for `{}`" .format(dps_name)) generate_dataentry_for(dps, 'dataentry.xlsx')
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Auto-validation started...") DEBUG = options.get('debug', False) now = timezone.now() # loop on non-validated DataRecord records = DataRecord.objects \ .filter(validation_status=DataRecord.NOT_VALIDATED, indicator__origin=Indicator.MANUAL) logger.info("On {on}, there are {nb} non-validated DataRecords".format( on=now, nb=records.count())) for dr in records: # continue if validation delay is not over if not dr.validation_period_is_over(): if DEBUG: logger.debug("Not validating {dr} until {date}".format( dr=dr, date=dr.validation_deadline.strftime('%c').decode( 'utf-8'))) continue # auto-validate dr.auto_validate(on=now) logger.debug("Auto-validated {}".format(dr)) logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Auto-validation started...") DEBUG = options.get('debug', False) now = timezone.now() # loop on non-validated DataRecord records = DataRecord.objects \ .filter(validation_status=DataRecord.NOT_VALIDATED, indicator__origin=Indicator.MANUAL) logger.info("On {on}, there are {nb} non-validated DataRecords" .format(on=now, nb=records.count())) for dr in records: # continue if validation delay is not over if not dr.validation_period_is_over(): if DEBUG: logger.debug("Not validating {dr} until {date}".format( dr=dr, date=dr.validation_deadline.strftime('%c') .decode('utf-8'))) continue # auto-validate dr.auto_validate(on=now) logger.debug("Auto-validated {}".format(dr)) logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error( "Unable to match an actual period from `{}`".format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = { i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis() } dhis_ids = list( set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': [ 'dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format(";".join([pe.dhis_strid for pe in periods])) ], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() yesterday = datetime.date.today() - datetime.timedelta(days=1) logger.info("Sending validation feedback for {}".format(yesterday)) start = datetime.datetime(*yesterday.timetuple()[:6]) end = datetime.datetime(*yesterday.timetuple()[:3], hour=23, minute=59, second=59) dhis_bot = Partner.dhis_bot() records = DataRecord.objects \ .exclude(created_by=dhis_bot) \ .exclude(validated_by__isnull=True) \ .filter(validated_on__gte=start, validated_on__lte=end) def summary_for(partner): pqs = records.filter(created_by=partner) return { 'partner': partner, 'nb_validations': pqs.count(), 'status': { s['validation_status']: { 'name': text_type( DataRecord.VALIDATION_STATUSES.get( s['validation_status'])), 'count': pqs.filter( validation_status=s['validation_status']).count(), 'all': pqs.filter(validation_status=s['validation_status']) } for s in pqs.values('validation_status') } } feedbacks = [ summary_for(Partner.objects.get(id=pid)) for pid in set( [r['created_by'] for r in records.values('created_by')]) if pid ] for ptnf in feedbacks: partner = ptnf['partner'] if not partner.email: continue ptnf.update({'yesterday': yesterday}) email_sent, x = send_validation_feedback_email(partner=partner, summary=ptnf) if email_sent: logger.info("Sent feedback to {}".format(partner)) else: logger.error("Unable to send feedback to {}".format(partner)) logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Updating cache for dashboard completeness...") root = Entity.get_root() periods = MonthPeriod.all_till_now() all_dps = root.get_children() all_entities = list(all_dps) + [root] indicators = Indicator.objects.all() all_indicators = list(indicators) + [None] nb_items = len(periods) * len(all_dps) * len(all_indicators) nb_ran = 0 for period in periods: # logger.debug("{}".format(period)) for dps in all_dps: # logger.debug("== {}".format(dps)) for indicator in all_indicators: nb_ran += 1 # logger.debug("==== {}".format(indicator)) params = { 'dps': dps, 'period': period, 'indicator': indicator } # existing cache 4months+ old are not regenerated if period <= periods[-4]: if cache_exists_for('completeness', **params): # logger.info("***** Skipping existing.") continue update_cached_data('completeness', **params) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/arrivals...") nb_items = len(periods) * len(all_dps) * len(indicators) nb_ran = 0 for period in periods: for entity in all_entities: for indicator in indicators: params = { 'entity': entity, 'period': period, 'indicator': indicator } if period <= periods[-4]: if cache_exists_for('section2-arrivals', **params): continue update_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/points") nb_items = len(periods) * len(all_dps) nb_ran = 0 for period in periods: for entity in all_entities: if period <= periods[-4]: if cache_exists_for('section2-points', **params): continue update_cached_data('section2-points', entity=entity, period=period) sys.stdout.write("{}/{} - {}%\r".format( nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("done.")
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() # options parsing self.debug = options.get('debug') update = options.get('update') period = MonthPeriod.get_or_none(options.get('period')) if period is None: logger.error("Unable to match an actual period from `{}`" .format(period)) if options.get('previous', False): periods = [] p = period while p > MonthPeriod.objects.all().first(): periods.append(p) if len(periods) >= NB_PREVIOUS_PERIODS: break p = p.previous() else: periods = [period] upath = '/analytics.json' indicators = {i.slug: (i.dhis_numerator_id, i.dhis_denominator_id) for i in Indicator.get_all_dhis()} dhis_ids = list(set([v[0] for v in indicators.values()] + [v[1] for v in indicators.values()])) drc = Entity.get_root() params = { 'dimension': ['dx:{}'.format(";".join(dhis_ids)), 'pe:{}'.format( ";".join([pe.dhis_strid for pe in periods]))], 'filter': 'ou:{}'.format(drc.dhis_id), 'displayProperty': 'NAME', 'outputIdScheme': 'ID', 'skipRounding': True, } logger.info(drc) if update or self.no_record_at(entity=drc, period=period): self.handle_record(get_dhis(path=upath, params=params), entity=drc, periods=periods) for dps in drc.get_children(): logger.info(dps) if not update and not self.no_record_at(entity=dps, period=period): continue dparams = copy.copy(params) dparams.update({'filter': 'ou:{}'.format(dps.dhis_id)}) self.handle_record(get_dhis(path=upath, params=dparams), entity=dps, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=dps, period=period): continue for zs in dps.get_children(): logger.info(zs) if not update and not self.no_record_at(entity=zs, period=period): continue zparams = copy.copy(params) zparams.update({'filter': 'ou:{}'.format(zs.dhis_id)}) self.handle_record(get_dhis(path=upath, params=zparams), entity=zs, periods=periods) # don't look for ZS if no data at DPS if self.no_record_at(entity=zs, period=period): continue for aire in zs.get_children(): logger.info(aire) if not update and not self.no_record_at(entity=aire, period=period): continue aparams = copy.copy(params) aparams.update({'filter': 'ou:{}'.format(aire.dhis_id)}) self.handle_record(get_dhis(path=upath, params=aparams), entity=aire, periods=periods)
def handle(self, *args, **options): # make sure we're at project root chdir_dmd() logger.info("Updating cache for dashboard completeness...") root = Entity.get_root() periods = MonthPeriod.all_till_now() all_dps = root.get_children() all_entities = list(all_dps) + [root] indicators = Indicator.objects.all() all_indicators = list(indicators) + [None] nb_items = len(periods) * len(all_dps) * len(all_indicators) nb_ran = 0 for period in periods: # logger.debug("{}".format(period)) for dps in all_dps: # logger.debug("== {}".format(dps)) for indicator in all_indicators: nb_ran += 1 # logger.debug("==== {}".format(indicator)) params = { 'dps': dps, 'period': period, 'indicator': indicator } # existing cache 4months+ old are not regenerated if period <= periods[-4]: if cache_exists_for('completeness', **params): # logger.info("***** Skipping existing.") continue update_cached_data('completeness', **params) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/arrivals...") nb_items = len(periods) * len(all_dps) * len(indicators) nb_ran = 0 for period in periods: for entity in all_entities: for indicator in indicators: params = { 'entity': entity, 'period': period, 'indicator': indicator } if period <= periods[-4]: if cache_exists_for('section2-arrivals', **params): continue update_cached_data('section2-arrivals', entity=entity, period=period, indicator=indicator) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("Updating cache for section2/points") nb_items = len(periods) * len(all_dps) nb_ran = 0 for period in periods: for entity in all_entities: if period <= periods[-4]: if cache_exists_for('section2-points', **params): continue update_cached_data('section2-points', entity=entity, period=period) sys.stdout.write("{}/{} - {}%\r" .format(nb_ran, nb_items, int(nb_ran / nb_items * 100))) sys.stdout.flush() logger.info("done.")