def main(configpath):
# Load the model configuration file and the data (observed cases)
config = load_config(configpath)
data = load_data(config)
useworldfile = config['worldfile']
if (not useworldfile):
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
plot_confidence(configpath, config, data, config['startdate'])
plot_confidence_alpha(configpath, config, data, config['startdate'])
def main(configpath):
# Load the model configuration file and the data (observed cases)
config = load_config(configpath)
data = load_data(config)
useworldfile = config['worldfile']
if (not useworldfile):
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
base = (os.path.split(configpath)[-1]).split('.')[0]
outpath = os.path.join(os.path.split(os.getcwd())[0], 'output', base)
plot_confidence(outpath, config, data, config['startdate'])
def plot_posterior(results, config, configpath, outpath, data, data_labels):
base = (os.path.split(configpath)[-1]).split('.')[0]
outputpath = os.path.join(outpath, base)
calmodes = [S_HOS, S_ICU, S_HOSCUM, S_DEAD]
o_indices = [O_HOS, O_ICU, O_HOSCUM, O_DEAD]
useworldfile = config['worldfile']
if (useworldfile):
calmodes = [S_DEAD]
o_indices = [O_DEAD]
else:
# check if I_HOS is present in the data, if not generate it
data = generate_hos_actual(data, config)
for i, calmode in enumerate(calmodes):
output_index = o_indices[i]
if calmode == S_HOS:
title = 'Hospitalized'
ymax = config['YMAX'] * config['hosfrac'] * 2
y_obs = data[:, I_HOS]
elif calmode == S_ICU:
title = 'ICU'
ymax = config['YMAX'] * config['hosfrac'] * config['ICufrac']
y_obs = data[:, I_ICU]
elif calmode == S_HOSCUM:
title = 'Hospitalized Cumulative'
ymax = config['YMAX'] * config['hosfrac'] * 2
y_obs = data[:, I_HOSCUM]
elif calmode == S_DEAD:
title = 'Fatalities'
ymax = config['YMAX'] * config['hosfrac'] * config['dfrac'] * 2
y_obs = data[:, I_DEAD]
else:
ymax = config['YMAX']
y_obs = np.full_like(data[:, 0], fill_value=np.nan)
title = ''
t_obs = data[:, data_labels.index('time')]
# Plot posterior
posterior = results
transparancy = max(min(5.0 / posterior.shape[0], 1), 0.01)
posterior_curves = np.array(
[member[output_index, :] for member in posterior]).T
time = results[:, O_TIME, :]
time_main = np.roll(time[0], int(time[0, 0]))
if time[0, 0] != 0.0:
time_main[int(time[0, 0]):] = np.nan
# Shift time solutions
posterior_shifted = np.zeros_like(posterior_curves)
for i, post in enumerate(posterior_curves.T):
shift = int(time[i, 0])
posterior_shifted[:, i] = np.roll(post, shift)
if time[0, 0] != 0.0:
posterior_shifted[shift:, i] = np.nan
post_mean = np.mean(posterior_shifted, axis=-1)
post_med = np.median(posterior_shifted, axis=-1)
color = 'blue'
plt.scatter(t_obs, y_obs, marker='o', c='k', label='Data')
plt.plot(time_main, posterior_shifted, alpha=transparancy, c=color)
plt.plot(time_main,
post_mean,
lw=2,
ls=':',
c='k',
label='Mean of posterior')
plt.plot(time_main, post_med, lw=2, c='k', label='Median of posterior')
plt.xlim(0, config['XMAX'])
plt.ylim(0, ymax)
plt.xlabel('Time [days]')
plt.ylabel('Number of cases')
plt.legend(loc='upper left')
plt.title(title + ' posterior ensemble')
plt.savefig('{}posterior_ensemble_{}.png'.format(outputpath, calmode),
dpi=300)
plt.xlim(0, time.max())
plt.savefig('{}posterior_ensemble_{}_longterm.png'.format(
outputpath, calmode),
dpi=300)
plt.ylim(0, posterior_curves.max())
plt.savefig('{}posterior_ensemble_{}_longterm_alt.png'.format(
outputpath, calmode),
dpi=300)
plt.close()
p_values = config['p_values']
p_array = []
steps = np.arange(1, time.max() + 1)
t_ind = [np.where(time_main == a)[0][0] for a in steps]
posterior_length = posterior_shifted.shape[1]
for post_day in posterior_shifted[t_ind, :]:
array_sorted = np.sort(post_day)
p_array.append(
[array_sorted[int(posterior_length * p)] for p in p_values])
h_pvalues = ['P' + str(int(100 * a)) for a in p_values]
header = 'time,mean,' + ','.join(h_pvalues) + ',observed'
p_array = np.asarray(p_array)
observed = np.pad(y_obs, (0, len(steps) - len(y_obs)),
mode='constant',
constant_values=np.nan)[:, None]
table = np.concatenate(
(steps[:, None], post_mean[t_ind, None], p_array, observed),
axis=1)
np.savetxt('{}_posterior_prob_{}_calibrated_on_{}.csv'.format(
outputpath, calmode, config['calibration_mode']),
table,
header=header,
delimiter=',',
comments='',
fmt='%.2f')
def run_esmda_model(base_filename, config, data, save_plots=1, save_files=1):
# concatenate additional column for actual observed hospitalized based
t_obs = data[:, I_TIME]
useworldfile = config['worldfile']
if not useworldfile:
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
# Run the forward model to obtain a prior ensemble of models
save_input_data(base_filename, data)
calibration_modes = get_calibration_modes(config['calibration_mode'])
observation_errors = get_calibration_errors(config['observation_error'])
calibration_mode = calibration_modes[0]
found = False
for i, calmode in enumerate(s_calmodes):
if calmode == calibration_mode:
print('ensemble fit on : ', calibration_modes[0], ' with error ',
observation_errors[0])
y_obs = data[:, i_calmodes[i]]
output_index = [o_calmodes[i]]
# error = np.ones_like(y_obs) * observation_errors[0]
if calmode == S_HOS or calmode == S_ICU:
error = y_obs * (observation_errors[0] / 100)
error[error <= 0] = 1
else:
error = np.ones_like(y_obs) * observation_errors[0]
found = True
if not found:
raise NotImplementedError
for ical, calibration_mode in enumerate(calibration_modes):
if ical > 0:
print('additional ensemble fit on : ', calibration_modes[ical],
' with error ', observation_errors[ical])
for i, calmode in enumerate(s_calmodes):
if calmode == calibration_mode:
y_obs2 = data[:, i_calmodes[i]]
output_index2 = [o_calmodes[i]]
# error2 = np.ones_like(y_obs2) * observation_errors[ical]
# error2[min(0,len(error2)-20):] *= 1
if calmode == S_HOS or calmode == S_ICU:
error2 = y_obs2 * (observation_errors[ical] / 100)
error2[error2 <= 0] = 1
else:
error2 = np.ones_like(
y_obs2) * observation_errors[ical]
y_obs = np.append(y_obs, y_obs2)
# output_index = [output_index[0], O_ICU]
output_index = np.append(output_index, output_index2)
error = np.append(error, error2)
found = True
if not found:
raise NotImplementedError
# Load inversion method and define forward model
try:
ni = config['esmda_iterations']
except KeyError:
ni = 8
im = InversionMethods(ni=ni)
forward = base_seir_model
np.random.seed(12345)
# Run the find_N function only if specified in the config file
try:
if config['find_N']:
nnew = find_N(config, data)
config['N'] = {
'type': 'uniform',
'min': 0.25 * nnew,
'max': 4 * nnew
}
except KeyError:
pass
# Parse the configuration json
ndata = np.size(data[:, 0])
m_prior, fwd_args = parse_config(config, ndata)
m_prior = reshape_prior(m_prior)
# Estimated uncertainty on the data
# sigma = np.sqrt(y_obs).clip(min=10)
# error = config['observation_error']
# if calibration_mode == S_HOS or calibration_mode == S_HOSCUM:
# error = 2 * error
# sigma = error * np.ones_like(y_obs)
sigma = error
add_arg = [fwd_args]
kwargs = {
't_obs': t_obs,
'output_index': output_index,
'G': map_model_to_obs,
'print_results': 1
}
results = im.es_mda(forward, m_prior, y_obs, sigma, *add_arg, **kwargs)
prior_and_posterior_results = save_and_plot_prior_and_posterior(
results, fwd_args, config, base_filename, t_obs, data,
calibration_mode, output_index, save_plots, save_files)
if save_files:
# if True:
# Add this to have a output folder in main directory
outpath = os.path.join(os.getcwd(), 'bin', 'output', base_filename)
# outpath = os.path.join(os.path.split(os.getcwd())[0], 'output', base_filename + '_output.h5')
save_results(results, fwd_args, config, outpath, data, mode='esmda')
try:
accc_timeinterval = config['ACCC_timeinterval']
accc_timestart = config['ACCC_timestart']
accc_step = config['ACCC_step']
accc_maxstep = config['ACCC_maxstep']
accc_step_sd = config['ACCC_step_sd']
accc_low = config['ACCC_low']
accc_slope = config['ACCC_slope']
accc_scale = config['ACCC_scale']
accc_cliplow = config['ACCC_cliplow']
accc_cliphigh = config['ACCC_cliphigh']
time_delay = config['time_delay']
hammer_icu = config['hammer_ICU']
hammer_slope = config['hammer_slope']
hammer_release = config['hammer_release']
hammer_alpha = config['hammer_alpha']
accc_results = apply_accc(base_filename,
results['M'][-1],
fwd_args,
accc_timeinterval,
accc_timestart,
accc_maxstep,
accc_step,
accc_step_sd,
accc_low,
accc_slope,
accc_scale,
hammer_icu,
hammer_slope,
hammer_alpha,
hammer_release,
accc_cliplow,
accc_cliphigh,
time_delay,
data_end=data[-1, 0])
# TODO: Add hammer results to h5 file and plotting
results['fw'][-1] = accc_results
save_and_plot_prior_and_posterior(results,
fwd_args,
config,
base_filename,
t_obs,
data,
calibration_mode,
output_index,
save_plots,
save_files,
plothammer=True)
add_hammer_to_results(accc_results, outpath, mode='esmda')
except KeyError:
pass
return prior_and_posterior_results, results
def main(configpath):
# Load the model configuration file and the data (observed cases)
config = load_config(configpath)
data = load_data(config)
outpath = config['outpath']
useworldfile = config['worldfile']
if (not useworldfile):
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
# Run the forward model to obtain a prior ensemble of models
save_input_data(configpath, outpath, data)
prior, time, prior_param, fwd_args = run_prior(config)
# Calibrate the model (as calibration data, we either use 'hospitalized' or 'dead')
calibration_mode = get_calibration_modes(config['calibration_mode'])[0]
if calibration_mode == 'dead':
data_index = I_DEAD
output_index = O_DEAD
elif calibration_mode == 'hospitalized':
data_index = I_HOS
output_index = O_HOS
elif calibration_mode == 'hospitalizedcum':
data_index = I_HOSCUM
output_index = O_HOSCUM
elif calibration_mode == 'ICU':
data_index = I_ICU
output_index = O_ICU
else:
raise NotImplementedError
plotplume = config['plot']['hindcast_plume']
p = calibrate_with_data(
prior, time, data, data_index,
output_index) # Posterior probability for each ensemble member
integrated_parameter_values = integrate_parameters(prior_param, p)
np.savetxt(outpath + "/integrated_parameter_values.csv",
integrated_parameter_values,
delimiter=";")
integrated_results = integrate_calibration(prior,
p) # Mean posterior model
# Create output of the calibration phase
plot_results(prior,
time,
configpath,
outpath,
config,
data,
integrated_results,
calibration_mode,
prior=True,
plot_plume=plotplume)
# Based on the model performance in the past, run models again, forecasting the future
forecast = rerun_model_with_alpha_uncertainty(prior_param, p, config,
fwd_args)
plot_results(forecast,
time,
configpath,
outpath,
config,
data,
cal_mode=config['calibration_mode'],
prior=False,
plot_plume=True)
plot_posterior(forecast, config, configpath, outpath, data, ['time'])
hospital_forecast_to_txt(forecast, time, configpath, outpath, config)
def main(configpath):
# Load the model configuration file and the data (observed cases)
config = load_config(configpath)
data = load_data(config)
# concatenate additional column for actual observed hospitalized based
t_obs = data[:, I_TIME]
useworldfile = config['worldfile']
if (not useworldfile):
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
# Run the forward model to obtain a prior ensemble of models
save_input_data(configpath, data)
calibration_modes = get_calibration_modes(config['calibration_mode'])
observation_errors = get_calibration_errors(config['observation_error'])
calibration_mode = calibration_modes[0]
found = False
for i, calmode in enumerate(s_calmodes):
if (calmode == calibration_mode):
print('ensemble fit on : ', calibration_modes[0], ' with error ',
observation_errors[0])
y_obs = data[:, i_calmodes[i]]
output_index = [o_calmodes[i]]
error = np.ones_like(y_obs) * observation_errors[0]
found = True
if not found:
raise NotImplementedError
for ical, calibration_mode in enumerate(calibration_modes):
if (ical > 0):
print('additional ensemble fit on : ', calibration_modes[ical],
' with error ', observation_errors[ical])
for i, calmode in enumerate(s_calmodes):
if (calmode == calibration_mode):
y_obs2 = data[:, i_calmodes[i]]
output_index2 = [o_calmodes[i]]
error2 = np.ones_like(y_obs2) * observation_errors[ical]
y_obs = np.append(y_obs, y_obs2)
# output_index = [output_index[0], O_ICU]
output_index = np.append(output_index, output_index2)
error = np.append(error, error2)
found = True
if not found:
raise NotImplementedError
# Load inversion method and define forward model
try:
ni = config['esmda_iterations']
except KeyError:
ni = 8
im = InversionMethods(ni=ni)
forward = base_seir_model
np.random.seed(12345)
# Parse the configuration json
m_prior, fwd_args = parse_config(config)
m_prior = reshape_prior(m_prior)
# Estimated uncertainty on the data
# sigma = np.sqrt(y_obs).clip(min=10)
# error = config['observation_error']
# if calibration_mode == S_HOS or calibration_mode == S_HOSCUM:
# error = 2 * error
# sigma = error * np.ones_like(y_obs)
sigma = error
add_arg = [fwd_args]
kwargs = {
't_obs': t_obs,
'output_index': output_index,
'G': map_model_to_obs,
'print_results': 1
}
results = im.es_mda(forward, m_prior, y_obs, sigma, *add_arg, **kwargs)
plot_prior_and_posterior(results, fwd_args, config, configpath, t_obs,
data, calibration_mode, output_index)
base = (os.path.split(configpath)[-1]).split('.')[0]
outpath = os.path.join(
os.path.split(os.getcwd())[0], 'output', base + '_output.h5')
save_results(results, fwd_args, config, outpath, data, mode='esmda')
# Check if we want to apply a 'hammer'
try:
hammer_icu = config['hammer_ICU']
hammer_alpha = config['hammer_alpha']
assert len(hammer_alpha) == 2
hammered_results = apply_hammer(results['fw'][-1],
results['M'][-1],
fwd_args,
hammer_icu,
hammer_alpha,
data_end=data[-1, 0])
# TODO: Add hammer results to h5 file and plotting
add_hammer_to_results(hammered_results, outpath, mode='esmda')
except KeyError as e:
pass # Don't apply hammer, you're done early!
def main(configpath):
# Load the model configuration file and the data (observed cases)
config = load_config(configpath)
data = load_data(config)
base_filename = (os.path.split(configpath)[-1]).split('.')[0]
useworldfile = config['worldfile']
if (not useworldfile):
data = generate_hos_actual(data, config)
else:
data = generate_zero_columns(data, config)
# Run the forward model to obtain a prior ensemble of models
save_input_data(base_filename, data)
ndata = np.size(data[:, 0])
prior, time, prior_param, fwd_args = run_prior(config, ndata)
# Calibrate the model (as calibration data, we either use 'hospitalized' or 'dead')
calibration_mode = get_calibration_modes(config['calibration_mode'])[0]
if calibration_mode == 'dead':
data_index = I_DEAD
output_index = O_DEAD
elif calibration_mode == 'hospitalized':
data_index = I_HOS
output_index = O_HOS
elif calibration_mode == 'hospitalizedcum':
data_index = I_HOSCUM
output_index = O_HOSCUM
elif calibration_mode == 'ICU':
data_index = I_ICU
output_index = O_ICU
else:
raise NotImplementedError
plotplume = config['plot']['hindcast_plume']
p = calibrate_with_data(
prior, time, data, data_index,
output_index) # Posterior probability for each ensemble member
integrated_results = integrate_calibration(prior,
p) # Mean posterior model
# Create output of the calibration phase
plot_results(prior,
time,
configpath,
config,
data,
integrated_results,
calibration_mode,
prior=True,
plot_plume=plotplume)
# Based on the model performance in the past, run models again, forecasting the future
forecast = rerun_model_with_alpha_uncertainty(prior_param, p, config,
fwd_args)
base = (os.path.split(configpath)[-1]).split('.')[0]
outbase = os.path.join(os.path.split(os.getcwd())[0], 'output', base)
plot_results(forecast,
time,
configpath,
config,
data,
cal_mode=config['calibration_mode'],
prior=False,
plot_plume=True)
plot_posterior(forecast, config, outbase, data, ['time'])
try:
hospital_forecast_to_txt(forecast, time, configpath, config, data)
except:
pass