def make_3d_pdf_weight(accepted, weights, num_bin_raw, C_limits, grid_kde,
output_folder):
left, right = C_limits[:, 0], C_limits[:, 1]
np.savetxt(os.path.join(output_folder, 'unique.txt'), accepted)
np.savetxt(os.path.join(output_folder, 'weights.txt'), weights)
print(f'Std of the accepted data is {np.std(accepted, axis=0)}')
# # # ##############################################################################
# # 3d histogram
print(f'2D raw marginals with {num_bin_raw} bins per dimension')
pp.calc_raw_joint_pdf(accepted, num_bin_raw, C_limits, weights)
pp.calc_marginal_pdf_raw(accepted, num_bin_raw, C_limits, output_folder,
weights)
# # # ##############################################################################
# # 3d kde
print(f'2D smooth marginals with {grid_kde} bins per dimension')
Z = gaussian_kde_scipy(accepted,
C_limits[:, 0],
C_limits[:, 1],
grid_kde,
weights=weights)
np.savez(os.path.join(output_folder, 'Z.npz'), Z=Z)
map_value = find_MAP_kde(Z, left, right)
np.savetxt(os.path.join(output_folder, 'map_value'), map_value)
pp.calc_marginal_pdf_smooth(Z,
grid_kde,
C_limits,
output_folder,
name='_dist')
def marginal(accepted,
C_limits,
num_bin_kde,
num_bin_raw,
folder,
mirror=False):
""" Generate marginal and 2d pdfs
:param accepted: list of accepted parameters
:param C_limits: list of bounds
:param num_bin_raw: number of bins in Nd histogram (per dimension)
:param num_bin_kde: number of points in Nd KDE
:param folder: path to output files
:param mirror: (boolen) if to use mirroring
"""
if not os.path.isdir(folder):
os.makedirs(folder)
##############################################################################
if num_bin_raw:
logging.info(
'2D raw marginals with {} bins per dimension'.format(num_bin_raw))
H, C_final_joint = pp.calc_raw_joint_pdf(accepted, num_bin_raw,
C_limits)
np.savetxt(os.path.join(folder, 'C_final_joint{}'.format(num_bin_raw)),
C_final_joint)
pp.calc_marginal_pdf_raw(accepted, num_bin_raw, C_limits, folder)
del H
# ##############################################################################
logging.info(
'2D smooth marginals with {} bins per dimension'.format(num_bin_kde))
if mirror:
mirrored_data, _ = pp.mirror_data_for_kde(accepted, C_limits[:, 0],
C_limits[:, 1])
print(
f"{len(mirrored_data) - len(accepted)} points were added to {len(accepted)} points"
)
Z = gaussian_kde_scipy(mirrored_data, C_limits[:, 0], C_limits[:, 1],
num_bin_kde)
else:
Z = kdepy_fftkde(accepted, C_limits[:, 0], C_limits[:, 1], num_bin_kde)
# Z = gaussian_kde_scipy(accepted, C_limits[:, 0], C_limits[:, 1], num_bin_kde)
C_final_smooth = find_MAP_kde(Z, C_limits[:, 0], C_limits[:, 1])
np.savetxt(os.path.join(folder, 'C_final_smooth' + str(num_bin_kde)),
C_final_smooth)
# np.savetxt(os.path.join(folder, 'mirrored_limits'), [left, right])
logging.info(
'Estimated parameters from joint pdf: {}'.format(C_final_smooth))
# ##############################################################################
np.savez(os.path.join(folder, 'Z.npz'), Z=Z)
# Z = np.load(os.path.join(folder, 'Z.npz'))['Z']
pp.calc_marginal_pdf_smooth(Z, num_bin_kde, C_limits, folder)
pp.calc_conditional_pdf_smooth(Z, folder)
make_pdfs.make_1d_pdf(accepted, num_bin_raw, C_limits, num_bin_kde, folder)
del Z
def make_3d_pdf(accepted,
num_bin_raw,
C_limits,
bound,
grid_kde,
output_folder,
dist_weight=None):
if dist_weight is None:
unique, weights = np.unique(accepted, axis=0, return_counts=True)
else:
unique, weights = accepted, dist_weight
np.savetxt(os.path.join(output_folder, 'unique.txt'), unique)
np.savetxt(os.path.join(output_folder, 'weights.txt'), weights)
print("Number of unique samples (without noise added) is", len(unique))
print(f'Std of the unique data is {np.std(unique, axis=0)}')
# # # ##############################################################################
# # 3d histogram
print(f'2D raw marginals with {num_bin_raw} bins per dimension')
pp.calc_raw_joint_pdf(unique, num_bin_raw, C_limits, weights)
pp.calc_marginal_pdf_raw(unique, num_bin_raw, C_limits, output_folder,
weights)
# # # ##############################################################################
print(f'2D smooth marginals with {grid_kde} bins per dimension')
(left, right) = bound
# # 3d kde
Z = gaussian_kde_scipy(unique, left, right, grid_kde, weights=weights)
np.savez(os.path.join(output_folder, 'Z.npz'), Z=Z)
map_value = find_MAP_kde(Z, left, right)
np.savetxt(os.path.join(output_folder, 'map_value'), map_value)
pp.calc_marginal_pdf_smooth(Z, grid_kde, bound, output_folder)
# 3d mirror
unique, weights = kde_2d_3d_mirror_data(unique, weights, left, right)
Z = gaussian_kde_scipy(unique, left, right, grid_kde, weights=weights)
pp.calc_marginal_pdf_smooth(Z,
grid_kde,
bound,
output_folder,
name='_mirror')
pp.calc_conditional_pdf_smooth(Z, output_folder, name='_mirror')
np.savez(os.path.join(output_folder, 'Z_mirror.npz'), Z=Z)
map_value = find_MAP_kde(Z, left, right)
np.savetxt(os.path.join(output_folder, 'map_value'), map_value)
def main():
path = {
'output': os.path.join('../', 'rans_output/'),
'valid_data': '../rans_ode/valid_data/'
}
print(path)
logging.basicConfig(format="%(levelname)s: %(name)s: %(message)s",
handlers=[
logging.FileHandler("{0}/{1}.log".format(
path['output'], 'ABClog_postprocess0005')),
logging.StreamHandler()
],
level=logging.DEBUG)
logging.info('\n############# POSTPROCESSING ############')
x_list = [0.3, 0.1, 0.05, 0.03, 0.01]
# x_list = [0.3]
num_bin_kde = 20
num_bin_raw = 10
C_limits = np.loadtxt(os.path.join(path['output'], 'C_limits_init'))
if len(np.array(C_limits).shape) < 2:
N_params = 1
else:
N_params = len(C_limits)
files_abc = glob.glob1(path['output'], "classic_abc*.npz")
files = [os.path.join(path['output'], i) for i in files_abc]
logging.info('Loading data')
dist_unsorted = load_dist(files)
ind = np.argsort(dist_unsorted[:, 0])
samples = load_c(files, N_params)[ind]
N_total, _ = samples.shape
dist = dist_unsorted[ind]
# ##################################################################################################################
# #
# # ################################################################################################################
logging.info('\n############# Regression ############')
# # regression_type = ['regression_dist', 'regression_full']
# regression_type = ['regression_full']
regression_type = ['regression_dist']
for type in regression_type:
path[type] = os.path.join(path['output'], type)
if not os.path.isdir(path[type]):
os.makedirs(path[type])
# if type == 'regression_full':
# sum_stat = load_sum_stat(files)[ind]
for x in x_list:
n = int(x * N_total)
folder = os.path.join(path[type], 'x_{}'.format(x * 100))
if not os.path.isdir(folder):
os.makedirs(folder)
logging.info('\n')
logging.info('Regression {}'.format(type))
logging.info(
'{} samples are taken for regression ({}% of {})'.format(
n, x * 100, N_total))
samples = samples[:n, :N_params]
dist = dist[:n]
# if type == 'regression_full':
# sum_stat = sum_stat[:n]
##########################################################################
if type == 'regression_dist':
new_samples, solution = regression(samples,
dist,
dist,
x=1,
folder=folder)
# else:
# Truth = sumstat.TruthData(valid_folder=path['valid_data'], case=input['case'])
# new_samples, solution = regression(samples, sum_stat - Truth.sumstat_true, dist, x=1)
limits = new_limits(new_samples, N_params)
np.savetxt(os.path.join(folder, 'reg_limits'), limits)
Z = kdepy_fftkde(new_samples, limits[:, 0], limits[:, 1],
num_bin_kde)
# Z, C_final_smooth = pp.gaussian_kde_scipy(new_samples, limits[:, 0], limits[:, 1], num_bin_kde_reg)
C_final_smooth = find_MAP_kde(Z, C_limits[:, 0], C_limits[:, 1])
logging.info('Estimated parameters from joint pdf: {}'.format(
C_final_smooth))
np.savetxt(
os.path.join(folder, 'C_final_smooth' + str(num_bin_kde)),
C_final_smooth)
np.savez(os.path.join(folder, 'Z.npz'), Z=Z)
pp.calc_marginal_pdf_smooth(Z, num_bin_kde, limits, folder)
pp.calc_conditional_pdf_smooth(Z, folder)
del Z
# for q in [0.05, 0.1, 0.25]:
# pp.marginal_confidence(N_params, folder, q)
# pp.marginal_confidence_joint(new_samples, folder, q)
logging.info('\n#############Done############')
def main(args):
# Initialization
if len(args) > 1:
input_path = args[1]
else:
input_path = os.path.join('../rans_output/', 'params.yml')
input = yaml.load(open(input_path, 'r'))
### Paths
basefolder = '../'
path = {
'output': os.path.join(basefolder, 'rans_output/'),
'plots': os.path.join(basefolder, 'rans_plots/'),
'valid_data': os.path.join(basefolder, 'rans_ode', 'valid_data')
}
# path = input['path']
path['calibration'] = os.path.join(path['output'], 'calibration/')
N_chains = input['parallel_threads']
logging.basicConfig(format="%(levelname)s: %(name)s: %(message)s",
handlers=[
logging.FileHandler("{0}/{1}.log".format(
path['output'], 'ABClog_postprocess')),
logging.StreamHandler()
],
level=logging.DEBUG)
logging.info('\n############# POSTPROCESSING CHAINS ############')
C_limits = np.loadtxt(os.path.join(path['calibration'], 'C_limits'))
N_params = len(C_limits)
files = np.empty(0)
for chain in range(N_chains):
files_onechain = glob.glob1(path['output'],
"chain{}_*.npz".format(chain))
files = np.hstack(
(files,
np.array(
[os.path.join(path['output'], i) for i in files_onechain])))
accepted = np.empty((0, N_params))
dist = np.empty((0, 1))
sum_stat = np.empty((0, len(np.load(files[0])['sumstat'][0])))
logging.info('Loading data')
for file in files:
logging.debug('loading {}'.format(file))
accepted = np.vstack((accepted, np.load(file)['C']))
sum_stat = np.vstack((sum_stat, np.load(file)['sumstat']))
dist = np.vstack((dist, np.load(file)['dist'].reshape((-1, 1))))
# data = np.hstack((accepted, dist)).tolist()
print(accepted.shape, sum_stat.shape, dist.shape)
logging.info('\n')
logging.info(
'\n############# MCMC-ABC ({} chains) ############'.format(N_chains))
folder = os.path.join(path['output'], 'chains')
if not os.path.isdir(folder):
os.makedirs(folder)
num_bin_kde = 50
num_bin_raw = 20
##############################################################################
logging.info(
'2D raw marginals with {} bins per dimension'.format(num_bin_raw))
H, C_final_joint = pp.calc_raw_joint_pdf(accepted, num_bin_raw, C_limits)
np.savetxt(os.path.join(folder, 'C_final_joint{}'.format(num_bin_raw)),
C_final_joint)
pp.calc_marginal_pdf_raw(accepted, num_bin_raw, C_limits, folder)
##############################################################################
logging.info(
'2D smooth marginals with {} bins per dimension'.format(num_bin_kde))
Z = kdepy_fftkde(accepted, C_limits[:, 0], C_limits[:, 1], num_bin_kde)
C_final_smooth = find_MAP_kde(Z, C_limits[:, 0], C_limits[:, 1])
np.savetxt(os.path.join(folder, 'C_final_smooth' + str(num_bin_kde)),
C_final_smooth)
logging.info(
'Estimated parameters from joint pdf: {}'.format(C_final_smooth))
np.savez(os.path.join(folder, 'Z.npz'), Z=Z)
pp.calc_marginal_pdf_smooth(Z, num_bin_kde, C_limits, folder)
pp.calc_conditional_pdf_smooth(Z, folder)