def __init__(self, directories: Directories):
self.directories = directories
# load settings
self.settings = load_regression_settings(directories.regression_version)
self.path_to_location_metadata = self.directories.get_location_metadata_file(
self.settings.location_set_version_id)
self.path_to_beta_scaling = self.directories.forecast_beta_scaling_dir
self.path_to_savefig = self.directories.forecast_diagnostic_dir
# load location metadata
self.location_metadata = pd.read_csv(self.path_to_location_metadata)
self.id2loc = self.location_metadata.set_index('location_id')[
'location_name'].to_dict()
# load locations
self.loc_ids = np.array([
file_name.split('_')[0]
for file_name in os.listdir(self.path_to_beta_scaling)
]).astype(int)
self.locs = np.array([
self.id2loc[loc_id]
for loc_id in self.loc_ids
])
# load data
self.scales_data = np.hstack([
pd.read_csv(self.directories.location_beta_scaling_file(loc_id))[
'beta_scales'
].values[:, None]
for loc_id in self.loc_ids
])
def __init__(self,
directories: Directories):
self.directories = directories
# load settings
self.settings = load_regression_settings(directories.regression_version)
self.path_to_location_metadata = self.directories.get_location_metadata_file(
self.settings.location_set_version_id)
self.path_to_betas_dir = self.directories.regression_beta_fit_dir
self.path_to_savefig = self.directories.regression_diagnostic_dir
# load location metadata
self.location_metadata = pd.read_csv(self.path_to_location_metadata)
self.id2loc = self.location_metadata.set_index('location_id')[
'location_name'].to_dict()
# load the beta data
self.df_beta = [
pd.read_csv(self.directories.get_draw_beta_fit_file(i))[[
'loc_id',
'date',
'days',
'beta',
'beta_pred'
]].dropna()
for i in range(self.settings.n_draws)
]
# location information
self.loc_ids = np.sort(list(self.df_beta[0]['loc_id'].unique()))
self.locs = np.array([
self.id2loc[loc_id]
for loc_id in self.loc_ids
])
self.num_locs = len(self.locs)
def __init__(self,
directories: Directories):
self.directories = directories
# load settings
self.settings = load_regression_settings(directories.regression_version)
self.path_to_location_metadata = self.directories.get_location_metadata_file(
self.settings.location_set_version_id)
self.path_to_coef_dir = self.directories.regression_coefficient_dir
self.path_to_savefig = self.directories.regression_diagnostic_dir
# load metadata
self.location_metadata = pd.read_csv(self.path_to_location_metadata)
self.id2loc = self.location_metadata.set_index('location_id')[
'location_name'].to_dict()
# load coef
df_coef = [
pd.read_csv(self.directories.get_draw_coefficient_file(i))
for i in range(self.settings.n_draws)
]
# organize information
self.covs = np.sort(list(self.settings.covariates.keys()))
self.covs = np.append(self.covs, 'intercept')
self.loc_ids = np.sort(list(df_coef[0]['group_id'].unique()))
self.locs = np.array([
self.id2loc[loc_id]
for loc_id in self.loc_ids
])
self.num_locs = len(self.locs)
# group coef data
self.coef_data = {}
for cov in self.covs:
coef_mat = np.vstack([
df[cov].values
for df in df_coef
])
coef_label = self.locs.copy()
coef_mean = coef_mat.mean(axis=0)
sort_idx = np.argsort(coef_mean)
self.coef_data[cov] = (coef_label[sort_idx], coef_mat[:, sort_idx])
def __init__(self, directories: Directories,
groups: list = None, exclude_groups: list = None,
col_group="loc_id", col_date='date'
):
self.directories = directories
self.col_group = col_group
self.col_date = col_date
self.groups = groups
if exclude_groups is not None:
for exclude_group in exclude_groups:
self.groups.remove(exclude_group)
self.data = {group: {
ODE_BETA_FIT: [],
COEFFICIENTS_FIT: [],
ODE_COMPONENTS_FORECAST: [],
OUTPUT_DRAWS_CASES: None,
OUTPUT_DRAWS_DEATHS: None,
OUTPUT_DRAWS_REFF: None
} for group in self.groups}
self.params_for_draws = []
# self.metadata = pd.read_csv("../../../data/covid/metadata-inputs/location_metadata_652.csv")
# TODO: change it for cluster
# self.metadata = pd.read_csv(directories.get_location_metadata_file(location_set_version_id=652))
# dictionary of location_id to name
# TODO: uncomment it for cluster to make Peng's part working
self.regression_settings = load_regression_settings(directories.regression_version)
self.forecast_settings = load_forecast_settings(directories.forecast_version)
self.location_metadata = pd.read_csv(
self.directories.get_location_metadata_file(
self.regression_settings.location_set_version_id)
)
self.id2loc = self.location_metadata.set_index('location_id')[
'location_name'].to_dict()
# read beta regression draws
for group in groups:
path_to_regression_draws_for_group = os.path.join(directories.regression_beta_fit_dir, str(group))
if os.path.isdir(path_to_regression_draws_for_group):
for filename in os.listdir(directories.regression_beta_fit_dir):
if filename.startswith("fit_draw_") and filename.endswith(".csv"):
draw_df = pd.read_csv(os.path.join(path_to_regression_draws_for_group, filename))
# It's assumed that draw_df contains only the `group` group exclusively
self.data[group][ODE_BETA_FIT].append(draw_df)
else:
continue
# Params and coefficients are commented out because Peng does not use them curently.
# # read coefficients draws
# for filename in os.listdir(directories.regression_coefficient_dir):
# if filename.startswith("coefficients_") and filename.endswith(".csv"):
# draw_df = pd.read_csv(os.path.join(directories.regression_coefficient_dir, filename))
# for group in self.groups:
# self.data[group][COEFFICIENTS_FIT].append(draw_df[draw_df['group_id'] == group])
# else:
# continue
#
# # read params draws
# for filename in os.listdir(directories.regression_parameters_dir):
# if filename.startswith("params_draw_") and filename.endswith(".csv"):
# draw_df = pd.read_csv(os.path.join(directories.regression_parameters_dir, filename))
# self.params_for_draws.append(draw_df)
# else:
# continue
# read components forecast
for group in groups:
path_to_compartments_draws_for_group = os.path.join(directories.forecast_component_draw_dir, str(group))
if os.path.isdir(path_to_compartments_draws_for_group):
for filename in os.listdir(path_to_compartments_draws_for_group):
if filename.startswith("draw_") and filename.endswith(".csv"):
draw_df = pd.read_csv(os.path.join(path_to_compartments_draws_for_group, filename))
self.data[group][ODE_COMPONENTS_FORECAST].append(draw_df)
else:
continue
else:
error_msg = f"ODE Components forecast for the group with {col_group} = {group} is not found"
print("Error: " + error_msg)
# raise FileNotFoundError(error_msg)
# read final draws
if os.path.isdir(directories.forecast_output_draw_dir):
for group in groups:
self.data[group][OUTPUT_DRAWS_CASES] = pd.read_csv(
os.path.join(directories.forecast_output_draw_dir, f"cases_{group}.csv"))
self.data[group][OUTPUT_DRAWS_DEATHS] = pd.read_csv(
os.path.join(directories.forecast_output_draw_dir, f"deaths_{group}.csv"))
self.data[group][OUTPUT_DRAWS_REFF] = pd.read_csv(
os.path.join(directories.forecast_output_draw_dir, f"reff_{group}.csv"))