def _export_log_info(df_exec, t_sec_1, t_sec_2):
# print(len(df_new), len(MODULES_NAME), len(df_new) == len(MODULES_NAME))
if len(df_exec) == len(MODULES_NAME):
print("EXPORTING LOG DETAILS")
details = system_details()
date_now = localdate(force_today=True)
machine = details["id"]
# Export timings per country
df_exec = df_exec.reset_index().assign(date=date_now, machine=machine)
df = obj_from_s3(LOG_GET_COUNTRIES)
df = df[df.date + df.machine != date_now + machine]
df = pd.concat([df, df_exec])
obj_to_s3(df, LOG_GET_COUNTRIES)
# Export machine info
data = obj_from_s3(LOG_MACHINES)
if machine not in data:
data = {**details, machine: details["info"]}
obj_to_s3(data, LOG_MACHINES)
# Export overall timing
report = {
"machine": machine,
"date": date_now,
"t_sec": t_sec_1,
"t_sec_retry": t_sec_2
}
df_new = pd.DataFrame([report])
df = obj_from_s3(LOG_GET_GLOBAL)
df = df[df.date + df.machine != date_now + machine]
df = pd.concat([df, df_new])
obj_to_s3(df, LOG_GET_GLOBAL)
def _load_modules_order(modules_name):
if len(modules_name) < 10:
return modules_name
df = obj_from_s3(LOG_GET_COUNTRIES)
# Filter by machine
# details = system_details()
# machine = details["id"]
# if machine in df.machine:
# df = df[df.machine == machine]
# df = pd.read_csv(os.path.join(PATHS.INTERNAL_OUTPUT_VAX_LOG_DIR, "get-data.csv"))
module_order_all = (df.sort_values("date").drop_duplicates(
subset=["module"],
keep="last").sort_values("execution_time (sec)",
ascending=False).module.tolist())
modules_name_order = [m for m in module_order_all if m in modules_name]
missing = [m for m in modules_name if m not in modules_name_order]
return modules_name_order + missing
def read(self, input_path: str):
if input_path.startswith("s3://"):
return obj_from_s3(input_path, parse_dates=[self.date])
return pd.read_csv(input_path, parse_dates=[self.date])
def read(self, input_path: str):
if input_path.startswith("s3://"):
return obj_from_s3(input_path)
return pd.read_csv(input_path)
def read(path, **kwargs):
if path.startswith("s3://"):
return obj_from_s3(path, **kwargs)
return pd.read_csv(path, **kwargs)
def from_ice(self):
"""Loads single CSV `location.csv` from S3 as DataFrame."""
path = f"{PATHS.S3_VAX_ICE_DIR}/{self.location}.csv"
_check_last_update(path, self.location)
df = obj_from_s3(path)
return df