map_layers_dict = {
layer: (local_path, layer_gdf)
for layer, local_path, layer_gdf in city_map_layers_to_minio.get_layers(tempdir,
secrets["minio"]["edge"]["access"],
secrets["minio"]["edge"]["secret"],
LAYER_FILES,
secrets["minio"]["lake"]["access"],
secrets["minio"]["lake"][
"secret"], )
}
logging.info("G[ot] layers")
logging.info("G[etting] SD Data")
sd_metric_data_df = service_delivery_latest_values_to_minio.get_data(
SD_METRIC_DATA_FILENAME,
secrets["minio"]["edge"]["access"],
secrets["minio"]["edge"]["secret"]
)
logging.info("G[ot] SD Data")
for layer_filename in CHOROPLETH_LAYERS:
source_layer, source_layer_index_col, null_value = CHOROPLETH_SOURCE_ATTRS[layer_filename]
_, data_gdf = map_layers_dict[source_layer]
logging.info(f"Generat[ing] metrics indexed for '{layer_filename}'")
indexed_df = generate_indexed_dict(sd_metric_data_df, source_layer_index_col, null_value)
logging.info(f"Generat[ed] metrics indexed for '{layer_filename}'")
logging.info(f"Merg[ing] metrics indexed for '{layer_filename}'")
indexed_gdf = generate_spatial_gdf(data_gdf, source_layer_index_col, indexed_df)
logging.info(f"Merg[ed] metrics indexed for '{layer_filename}'")
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s-%(module)s.%(funcName)s [%(levelname)s]: %(message)s')
# Loading secrets
SECRETS_PATH_VAR = "SECRETS_PATH"
if SECRETS_PATH_VAR not in os.environ:
sys.exit(-1)
secrets_path = os.environ["SECRETS_PATH"]
secrets = json.load(open(secrets_path))
logging.info("Fetch[ing] SD data...")
# Getting service delivery time series
sd_timeseries_df = service_delivery_latest_values_to_minio.get_data(
service_delivery_latest_values_to_minio.SD_DATA_FILENAME,
secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"]
)
sd_timeseries_df[DATE_COL] = pandas.to_datetime(sd_timeseries_df[DATE_COL])
logging.info("...Fetch[ed] SD data.")
features = sd_timeseries_df[FEATURE_COL].unique()
for feature in features:
feature_clauses = feature.split("-")
feature_clauses = [REMAP_DICT.get(feature_val, feature_val) for feature_val in feature_clauses]
city_file_prefix = feature_clauses[0]
directorate_file_prefix = feature_clauses[1] if len(feature_clauses) > 1 else None
department_file_prefix = feature_clauses[2] if len(feature_clauses) > 2 else None
if ((directorate_file_prefix and directorate_file_prefix in SKIP_LIST) or
(department_file_prefix and department_file_prefix in SKIP_LIST)):
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s-%(module)s.%(funcName)s [%(levelname)s]: %(message)s')
# Loading secrets
SECRETS_PATH_VAR = "SECRETS_PATH"
if SECRETS_PATH_VAR not in os.environ:
sys.exit(-1)
secrets_path = os.environ["SECRETS_PATH"]
secrets = json.load(open(secrets_path))
logging.info("Fetch[ing] SD data...")
request_data_df = service_delivery_latest_values_to_minio.get_data(
SD_DEPT_METRICS,
secrets["minio"]["edge"]["access"], secrets["minio"]["edge"]["secret"]
)
logging.info("...Fetch[ed] SD data.")
depts = DEPARTMENTS + [CITY_VALUE]
for direct, dept in depts:
logging.info(f"Generat[ing] plot for '{direct}'-'{dept}'")
logging.debug(f"service_request_df.shape={request_data_df.shape}")
logging.info(f"Filter[ing] data...")
filtered_df = (request_data_df.query(
f"(department == @dept)"
) if dept else request_data_df).copy()
logging.info(f"...Filter[ed] data")
logging.debug(f"filtered_df.shape={filtered_df.shape}")