map_layers_dict = { layer: (local_path, layer_gdf) for layer, local_path, layer_gdf in city_map_layers_to_minio.get_layers(tempdir, secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"], LAYER_FILES, secrets["minio"]["lake"]["access"], secrets["minio"]["lake"][ "secret"], ) } logging.info("G[ot] layers") logging.info("G[etting] SD Data") sd_metric_data_df = service_delivery_latest_values_to_minio.get_data( SD_METRIC_DATA_FILENAME, secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"] ) logging.info("G[ot] SD Data") for layer_filename in CHOROPLETH_LAYERS: source_layer, source_layer_index_col, null_value = CHOROPLETH_SOURCE_ATTRS[layer_filename] _, data_gdf = map_layers_dict[source_layer] logging.info(f"Generat[ing] metrics indexed for '{layer_filename}'") indexed_df = generate_indexed_dict(sd_metric_data_df, source_layer_index_col, null_value) logging.info(f"Generat[ed] metrics indexed for '{layer_filename}'") logging.info(f"Merg[ing] metrics indexed for '{layer_filename}'") indexed_gdf = generate_spatial_gdf(data_gdf, source_layer_index_col, indexed_df) logging.info(f"Merg[ed] metrics indexed for '{layer_filename}'")
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s-%(module)s.%(funcName)s [%(levelname)s]: %(message)s') # Loading secrets SECRETS_PATH_VAR = "SECRETS_PATH" if SECRETS_PATH_VAR not in os.environ: sys.exit(-1) secrets_path = os.environ["SECRETS_PATH"] secrets = json.load(open(secrets_path)) logging.info("Fetch[ing] SD data...") # Getting service delivery time series sd_timeseries_df = service_delivery_latest_values_to_minio.get_data( service_delivery_latest_values_to_minio.SD_DATA_FILENAME, secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"] ) sd_timeseries_df[DATE_COL] = pandas.to_datetime(sd_timeseries_df[DATE_COL]) logging.info("...Fetch[ed] SD data.") features = sd_timeseries_df[FEATURE_COL].unique() for feature in features: feature_clauses = feature.split("-") feature_clauses = [REMAP_DICT.get(feature_val, feature_val) for feature_val in feature_clauses] city_file_prefix = feature_clauses[0] directorate_file_prefix = feature_clauses[1] if len(feature_clauses) > 1 else None department_file_prefix = feature_clauses[2] if len(feature_clauses) > 2 else None if ((directorate_file_prefix and directorate_file_prefix in SKIP_LIST) or (department_file_prefix and department_file_prefix in SKIP_LIST)):
if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG, format='%(asctime)s-%(module)s.%(funcName)s [%(levelname)s]: %(message)s') # Loading secrets SECRETS_PATH_VAR = "SECRETS_PATH" if SECRETS_PATH_VAR not in os.environ: sys.exit(-1) secrets_path = os.environ["SECRETS_PATH"] secrets = json.load(open(secrets_path)) logging.info("Fetch[ing] SD data...") request_data_df = service_delivery_latest_values_to_minio.get_data( SD_DEPT_METRICS, secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"] ) logging.info("...Fetch[ed] SD data.") depts = DEPARTMENTS + [CITY_VALUE] for direct, dept in depts: logging.info(f"Generat[ing] plot for '{direct}'-'{dept}'") logging.debug(f"service_request_df.shape={request_data_df.shape}") logging.info(f"Filter[ing] data...") filtered_df = (request_data_df.query( f"(department == @dept)" ) if dept else request_data_df).copy() logging.info(f"...Filter[ed] data") logging.debug(f"filtered_df.shape={filtered_df.shape}")