def compare_freyberg_local3():
df_l = pd.read_csv(
os.path.join("ies_freyberg", "master_local3", "pest_local.6.par.csv"))
df_f = pd.read_csv(
os.path.join("ies_freyberg", "master_local3_base",
"pest_base.6.par.csv"))
pst = pyemu.Pst(os.path.join("ies_freyberg", "template", "pest.pst"))
df_l.columns = df_l.columns.str.lower()
df_f.columns = df_f.columns.str.lower()
groups = ["w1", "r1"]
par = pst.parameter_data
# pars = par.loc[par.pargp.apply(lambda x: x in groups)]
# df_l = df_l.loc[:,pars]
# df_f = df_f.loc[:,pars]
for group, title in zip(groups, ["future pumping", "future recharge"]):
fig = plt.figure(figsize=(10, 10))
ax1, ax2 = plt.subplot(211), plt.subplot(212)
fig.suptitle(title +
" parameters with (blue) and w/o (green) localization")
ax1.set_title("mean")
ax2.set_title("standard deviation")
for ax in zip(groups, [ax1, ax2]):
pars = par.loc[par.pargp == group, "parnme"]
df_lg = df_l.loc[:, pars]
df_fg = df_f.loc[:, pars]
mn_l, st_l = df_lg.mean(), df_lg.std()
mn_f, st_f = df_fg.mean(), df_fg.std()
print(mn_l, mn_f)
print(st_l, st_f)
mn_l.hist(ax=ax1, facecolor='b', alpha=0.5, normed=True)
mn_f.hist(ax=ax1, facecolor='g', alpha=0.5, normed=True)
st_l.hist(ax=ax2, facecolor='b', alpha=0.5, normed=True)
st_f.hist(ax=ax2, facecolor='g', alpha=0.5, normed=True)
ax2.set_yticklabels([])
ax1.set_yticklabels([])
plt.show()
def setup():
if os.path.exists("master"):
shutil.rmtree("master")
#shutil.copytree(base_d,"master")
pst = pyemu.Pst(os.path.join(base_d, "pest.pst"))
pst.pestpp_options = {}
pst.control_data.noptmax = 3
#pst.pestpp_options["ies_subset_size"] = 10
nreal_per = 1
dfs = []
obsvals = np.linspace(0.0, 5.0, 20)
for obsval in obsvals:
obs = pst.observation_data.obsval.copy()
obs.loc[oname] = obsval
# for i in range(nreal_per):
# dfs.append(obs)
for obsval2 in obsvals:
obs1 = obs.copy()
obs1.loc["h01_04"] = obsval2
dfs.append(obs)
df = pd.concat(dfs, axis=1).T
df.index = np.arange(df.shape[0])
df.to_csv(os.path.join(base_d, "obs.csv"))
pst.pestpp_options["ies_obs_en"] = "obs.csv"
pst.pestpp_options["ies_num_reals"] = df.shape[0]
pst.write(os.path.join(base_d, "pest_pareto.pst"))
#pyemu.os_utils.run("pestpp-ies pest.pst /h :4004",cwd="master")
pyemu.os_utils.start_slaves("template",
"pestpp-ies",
"pest_pareto.pst",
num_slaves=20,
master_dir="master")
plot()
def tie_by_group_test():
model_d = "ies_10par_xsec"
local=True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local=False
t_d = os.path.join(model_d,"template")
m_d = os.path.join(model_d,"master_tie_by_group")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
par = pst.parameter_data
tied_names = pst.adj_par_names[:3]
par.loc[tied_names[1:3],"partrans"] = "tied"
par.loc[tied_names[1:3],"partied"] = tied_names[0]
pst.pestpp_options = {}
pst.pestpp_options["ies_num_reals"] = 10
pst.pestpp_options["ies_lambda_mults"] = 1.0
pst.pestpp_options["lambda_scale_fac"] = 1.0
pst.pestpp_options["tie_by_group"] = True
pst.control_data.noptmax = 2
pst.write(os.path.join(t_d,"pest_tied.pst"))
pyemu.os_utils.start_slaves(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d,
slave_root=model_d,local=local,port=port)
df = pd.read_csv(os.path.join(m_d,"pest_tied.{0}.par.csv".format(pst.control_data.noptmax)),index_col=0)
df.columns = df.columns.str.lower()
print(df.loc[:,tied_names].std(axis=1).apply(np.abs).max())
assert df.loc[:,tied_names].std(axis=1).apply(np.abs).max() < 1.0e-8
df.to_csv(os.path.join(t_d,"sweep_in.csv"))
pyemu.os_utils.start_slaves(t_d, exe_path.replace("-ies","-swp"), "pest_tied.pst", 5, master_dir=m_d,
slave_root=model_d,local=local,port=port)
pyemu.os_utils.start_slaves(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d,
slave_root=model_d,local=local,port=port)
jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb"))
assert jco.shape[1] == 2,jco.shape
def change_limit_test():
import numpy as np
import pyemu
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
#print(pst.parameter_data)
cols = [
"parval1", "rel_upper", "rel_lower", "fac_upper", "fac_lower",
"chg_upper", "chg_lower"
]
pst.control_data.relparmax = 3
pst.control_data.facparmax = 3
par = pst.parameter_data
par.loc[:, "parval1"] = 1.0
df = pst.get_par_change_limits()
assert df.rel_upper.mean() == 4.0
assert df.rel_lower.mean() == -2.0
assert df.fac_upper.mean() == 3.0
assert np.abs(df.fac_lower.mean() - 0.33333) < 1.0e-3
pst.control_data.facorig = 2.0
par.loc[:, "partrans"] = "none"
df = pst.get_par_change_limits()
assert df.rel_upper.mean() == 8.0
assert df.rel_lower.mean() == -4.0
assert df.fac_upper.mean() == 6.0
assert np.abs(df.fac_lower.mean() - 0.66666) < 1.0e-3
pst.control_data.facorig = 0.001
par.loc[:, "partrans"] = "none"
par.loc[:, "parval1"] = -1.0
df = pst.get_par_change_limits()
#print(df.loc[:, cols])
assert df.rel_upper.mean() == 2.0
assert df.rel_lower.mean() == -4.0
assert df.fac_lower.mean() == -3.0
assert np.abs(df.fac_upper.mean() + 0.33333) < 1.0e-3
print(df.loc[:, ["eff_upper", "eff_lower"]])
print(df.loc[:, cols])
def secondary_marker_test():
t_d = os.path.join("secondary_marker_test","template")
tpl_file = os.path.join(t_d,"par.dat.tpl")
with open(tpl_file,'w') as f:
f.write("ptf ~\n")
f.write("~ p1 ~\n")
tpl_file = "par.dat.tpl"
b_d = os.getcwd()
os.chdir(t_d)
try:
ins_files = [f for f in os.listdir(".") if f.endswith(".ins")]
with open("forward_run.py",'w') as f:
f.write("import shutil\n")
for ins_file in ins_files:
out_file = ins_file.replace(".ins","")
f.write("shutil.copy2('{0}','{1}')\n".format(out_file+"_bak",out_file))
for ins_file in ins_files:
shutil.copy2(out_file+"_bak",out_file)
pst = pyemu.Pst.from_io_files(tpl_file,tpl_file.replace(".tpl",""),
ins_file,ins_file.replace(".ins",""))
pst.control_data.noptmax = 0
pst.pestpp_options["additional_ins_delimiters"] = "|"
pst.model_command = "python forward_run.py"
pst.write(os.path.join("test.pst"))
pyemu.os_utils.run("{0} test.pst".format(exe_path))
pst = pyemu.Pst("test.pst")
assert pst.res is not None
d = pst.res.loc[pst.obs_names,"modelled"] - pst.observation_data.loc[pst.obs_names,"obsval"]
l2_d = (d.apply(np.abs)**2).sum()
except Exception as e:
os.chdir(b_d)
raise Exception(e)
os.chdir(b_d)
def predvar1b_test():
out_files = [
os.path.join(verf_dir, f) for f in os.listdir(verf_dir)
if f.endswith(".out") and "ident" not in f
]
pv1b_results = {}
for out_file in out_files:
pred_name = os.path.split(out_file)[-1].split('.')[0]
f = open(out_file, 'r')
for _ in range(3):
f.readline()
arr = np.loadtxt(f)
pv1b_results[pred_name] = arr
pst = pyemu.Pst(ord_base + ".pst")
omitted_parameters = [
pname for pname in pst.parameter_data.parnme if pname.startswith("wf")
]
la_ord_errvar = pyemu.ErrVar(jco=ord_base + ".jco",
predictions=predictions,
omitted_parameters=omitted_parameters,
verbose=False)
df = la_ord_errvar.get_errvar_dataframe(np.arange(36))
max_idx = 13
idx = np.arange(max_idx)
for ipred, pred in enumerate(predictions):
arr = pv1b_results[pred][:max_idx, :]
first = df[("first", pred)][:max_idx]
second = df[("second", pred)][:max_idx]
third = df[("third", pred)][:max_idx]
first_diff = (np.abs(arr[:, 1] - first)).sum()
second_diff = (np.abs(arr[:, 2] - second)).sum()
third_diff = (np.abs(arr[:, 3] - third)).sum()
print(pred, first_diff, second_diff, third_diff)
assert first_diff < 1.5
assert second_diff < 1.5
assert third_diff < 1.5
def invest():
m_d = os.path.join("henry", "henry_master_riskobj_all_once")
pst = pyemu.Pst(os.path.join(m_d, "henry.pst"))
onames = pst.nnz_obs_names
df = pd.read_csv(os.path.join(m_d, "henry.0.nested.obs_stack.csv"))
df.loc[:, "member"] = df.real_name.apply(lambda x: x.split("||")[1])
df.loc[:, "real"] = df.real_name.apply(lambda x: x.split("||")[0])
mnames = df.member.unique()
mnames.sort()
rnames = df.real.unique()
rnames.sort()
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
with PdfPages(os.path.join(m_d, "stack_summary.pdf")) as pdf:
for oname in onames:
fig, ax = plt.subplots(1, 1, figsize=(6, 6))
for rname in rnames:
dfm = df.loc[df.real == rname]
dfm.loc[:, oname].hist(ax=ax, alpha=0.5, bins=20)
ax.set_title(oname)
pdf.savefig()
plt.close(fig)
def write_par_sum(pst_file):
pst = pyemu.Pst(pst_file)
par = pst.parameter_data
par.loc[par.pargp.str.startswith("wel"), "pargp"] = "wel"
def group_namer(grp):
name = ''
if "layer" in grp:
name += "Layer " + grp.split('layer')[1][0]
if "gr" in grp:
name += " grid-scale"
elif "pp" in grp:
name += " pilot points"
elif "const" in grp:
name += " constant"
if "_k_" in grp:
name += " HK"
elif "k33" in grp:
name += " VK"
elif "ss" in grp:
name += " SS"
elif "sy" in grp:
name += " SY"
elif "rch" in grp:
name += " recharge"
if "sfr" in grp:
name = " SFR stream-bed conductance"
if "twel" in grp:
name = "temporal wel flux constants"
#elif "wel" in grp:
# name = "grid-scale wel flux for stress period " + grp.split('_')[1]
elif "wel" in grp:
name = "grid-scale wel flux"
return name
ugrps = pst.parameter_data.pargp.unique()
name_dict = {ug: group_namer(ug) for ug in ugrps}
pst.write_par_summary_table(os.path.split(pst_file)[0] + "_par_sum.tex",
group_names=name_dict)
def add_obs_test():
import os
import pyemu
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
nobs = pst.nobs
ins_file = os.path.join("temp", "crap.out.ins")
out_file = os.path.join("temp", "crap.out")
oval = 1234.56
with open(ins_file, 'w') as f:
f.write("pif ~\n")
# f.write(" ~junk1 ~\n")
# f.write(" ~ {0} ~\n".format(pst.parameter_data.parnme[0]))
f.write("l1 w !{0}!\n".format("crap1"))
with open(out_file, "w") as f:
f.write("junk1 {0:8.2f} \n".format(oval))
pst.add_observations(ins_file, out_file, pst_path="temp")
assert nobs + 1 == pst.nobs
assert "crap1" in pst.observation_data.obsnme
assert os.path.join("temp",
"crap.out") in pst.output_files, str(pst.output_files)
assert os.path.join("temp", "crap.out.ins") in pst.instruction_files
print(pst.observation_data.loc["crap1", "obsval"], oval)
def geostat_prior_builder_test():
import os
import numpy as np
import pyemu
pst_file = os.path.join("pst", "pest.pst")
pst = pyemu.Pst(pst_file)
tpl_file = os.path.join("utils", "pp_locs.tpl")
str_file = os.path.join("utils", "structure.dat")
cov = pyemu.helpers.geostatistical_prior_builder(pst_file,
{str_file: tpl_file})
d1 = np.diag(cov.x)
df = pyemu.gw_utils.pp_tpl_to_dataframe(tpl_file)
df.loc[:, "zone"] = np.arange(df.shape[0])
gs = pyemu.geostats.read_struct_file(str_file)
cov = pyemu.helpers.geostatistical_prior_builder(pst_file, {gs: df},
sigma_range=4)
nnz = np.count_nonzero(cov.x)
assert nnz == pst.npar
d2 = np.diag(cov.x)
assert np.array_equiv(d1, d2)
def rectify_pgroup_test():
import os
import pyemu
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
npar = pst.npar
tpl_file = os.path.join("temp", "crap.in.tpl")
with open(tpl_file, 'w') as f:
f.write("ptf ~\n")
f.write(" ~junk1 ~\n")
f.write(" ~ {0} ~\n".format(pst.parameter_data.parnme[0]))
# print(pst.parameter_groups)
pst.add_parameters(tpl_file, "crap.in", pst_path="temp")
# print(pst.parameter_groups)
pst.rectify_pgroups()
# print(pst.parameter_groups)
pst.parameter_groups.loc["pargp", "inctyp"] = "absolute"
print(pst.parameter_groups)
pst.write(os.path.join('temp', "test.pst"))
print(pst.parameter_groups)
def from_names_test():
import os
import pyemu
rnames = ["row_{0}".format(i) for i in range(20)]
cnames = ["col_{0}".format(i) for i in range(40)]
m = pyemu.Matrix.from_names(rnames,cnames)
assert m.shape[0] == len(rnames)
assert m.shape[1] == len(cnames)
pst_name = os.path.join("pst","pest.pst")
pst = pyemu.Pst(pst_name)
j = pyemu.Jco.from_pst(pst)
jj = pyemu.Jco.from_pst(pst_name)
assert j.shape == jj.shape
c = pyemu.Cov.from_names(rnames,cnames)
assert type(c) == pyemu.Cov
def prep():
if os.path.exists("master"):
shutil.rmtree("master")
shutil.copytree("template", "master")
pst = pyemu.Pst(os.path.join("master", "pest.pst"))
pst.pestpp_options = {}
pst.pestpp_options["ies_num_reals"] = 10
pst.pestpp_options["ies_save_binary"] = True
pst.pestpp_options["ies_lambda_mults"] = [0.1, 1.0]
pst.pestpp_options["lambda_scale_fac"] = [0.9, 1.0]
pst.pestpp_options["ies_subset_size"] = 3
pst.pestpp_options["par_sigma_range"] = 20
pst.pestpp_options["ies_parameter_csv"] = "par.jcb"
pst.pestpp_options["ies_observation_csv"] = "obs.jcb"
pst.pestpp_options["ies_obs_restart_csv"] = "restart_obs.jcb"
pst.pestpp_options["ies_use_approx"] = "true"
pst.svd_data.eigthresh = 1.0e-5
pst.svd_data.maxsing = 1.0e+6
pst.control_data.noptmax = 1
pst.write(os.path.join("master", "pest.pst"))
def mf6_v5_glm_test():
model_d = "mf6_freyberg"
local=True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local=False
t_d = os.path.join(model_d,"template")
m_d = os.path.join(model_d,"master_glm")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst"))
m_d = os.path.join(model_d,"master_glm")
pyemu.os_utils.start_workers(t_d, "pestpp-glm", "freyberg6_run_glm.pst",
num_workers=15, master_dir=m_d,worker_root=model_d,
port=port)
oe_file = os.path.join(m_d,"freyberg6_run_glm.post.obsen.csv")
assert os.path.exists(oe_file)
oe = pd.read_csv(oe_file)
assert oe.shape[0] == pst.pestpp_options["glm_num_reals"],"{0},{1}".\
format(oe.shape[0],pst.pestpp_options["glm_num_reals"])
def pnulpar_test():
pst = pyemu.Pst(ord_base + ".pst")
# load the pnulpar projected ensemble
pnul_en = pyemu.ParameterEnsemble(pst)
pnul_en.read_parfiles_prefix(
os.path.join(verf_dir, "proj_par_draws", "draw_"))
pnul_en.loc[:, "fname"] = pnul_en.index
pnul_en.index = pnul_en.fname.apply(
lambda x: str(int(x.split('.')[0].split('_')[-1])))
f = pnul_en.pop("fname")
mc = pyemu.MonteCarlo(jco=ord_base + ".jco")
mc.parensemble.read_parfiles_prefix(
os.path.join(verf_dir, "prior_par_draws", "draw_"))
mc.parensemble.loc[:, "fname"] = mc.parensemble.index
mc.parensemble.index = mc.parensemble.fname.apply(
lambda x: str(int(x.split('.')[0].split('_')[-1])))
f = mc.parensemble.pop("fname")
en = mc.project_parensemble(nsing=1, inplace=False)
diff = 100 * (np.abs(pnul_en - en) / en)
assert max(diff.max()) < 1.0e-3
def sen_plusplus_test():
model_d = "ies_10par_xsec"
local=True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local=False
t_d = os.path.join(model_d,"template")
m_d = os.path.join(model_d,"master_sen_plusplus")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
pst.pestpp_options = {}
pst.pestpp_options["gsa_method"] = "morris"
pst.pestpp_options["gsa_sobol_samples"] = 50
pst.pestpp_options["gsa_sobol_par_dist"] = "unif"
pst.pestpp_options["gsa_morris_r"] = 4
pst.pestpp_options["gsa_morris_p"] = 5
pst.pestpp_options["gsa_morris_delta"] = 2
pst.write(os.path.join(t_d,"pest_sen.pst"))
pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_sen.pst", 5, master_dir=m_d,
worker_root=model_d,local=local,port=port)
def extract_and_plot_solution():
m_d = os.path.join("henry", "henry_master_riskobj_all_once")
#m_d = os.path.join("henry","henry_master_95_single_once")
pst = pyemu.Pst(os.path.join(m_d, "henry.pst"))
df_arc = pd.read_csv(os.path.join(m_d, "henry.pareto.archive.summary.csv"))
df_arc = df_arc.loc[df_arc.generation == df_arc.generation.max(), :]
df_arc.loc[:, "member"] = df_arc.member.str.lower()
df_arc.sort_values(by="ar_dist", ascending=False, inplace=True)
df_dv = pd.read_csv(os.path.join(m_d, "henry.archive.dv_pop.csv"))
#dv_vals = df_dv.loc[df_arc.member.iloc[0],:]
#df_dv = df_dv.loc[df_dv._risk_>0.95,:]
df_dv.sort_values(by="_risk_", ascending=False, inplace=True)
print(df_dv.loc[:, "ar_rate"])
pst.parameter_data.loc[:, "parval1"] = df_dv.loc[df_dv.index[2],
pst.par_names]
pst.control_data.noptmax = 0
pst.write(os.path.join(m_d, "test.pst"))
pyemu.os_utils.run("{0} test.pst".format(exe_path), cwd=m_d)
run_and_plot_results(m_d)
plot_domain(m_d, include_pred=True)
def emp_cov_test():
import os
import numpy as np
import pyemu
pst = pyemu.Pst(os.path.join("en", "pest.pst"))
cov = pyemu.Cov.from_binary(os.path.join("en", "cov.jcb"))
print(pst.npar, cov.shape)
num_reals = 10000
pe_eig = pyemu.ParameterEnsemble.from_gaussian_draw(pst,
cov=cov,
num_reals=num_reals,
factor="eigen")
emp_cov = pe_eig.covariance_matrix()
assert isinstance(emp_cov, pyemu.Cov)
assert emp_cov.row_names == pst.adj_par_names
cov_df = cov.to_dataframe()
emp_df = emp_cov.to_dataframe()
for p in pst.adj_par_names:
print(p, cov_df.loc[p, p], emp_df.loc[p, p])
diff = np.diag(cov.x) - np.diag(emp_cov.x)
print(diff.max())
assert diff.max() < 0.5, diff.max()
def start_workers_for_debug(with_master=True):
t_d = os.path.join("henry", "henry_template")
m_d = os.path.join("henry", "henry_master_chance_risk_obj")
if with_master:
if os.path.exists(m_d):
shutil.rmtree(m_d)
shutil.copytree(t_d, m_d)
pst = pyemu.Pst(os.path.join(m_d, "henry.pst"))
pst.control_data.noptmax = 200
pst.pestpp_options["opt_par_stack"] = "prior.jcb"
pst.pestpp_options["opt_stack_size"] = 30
pst.pestpp_options["opt_recalc_chance_every"] = 1000
pst.pestpp_options["mou_population_size"] = 100
pst.pestpp_options["opt_chance_points"] = "all"
pst.pestpp_options["opt_risk"] = 0.70
pst.write(os.path.join(m_d, "henry.pst"))
pyemu.os_utils.start_workers(t_d,
exe_path,
"henry.pst",
num_workers=12,
worker_root="henry",
port=4004)
def run_problem_chance(test_case="zdt1",
pop_size=100,
noptmax=10,
stack_size=30,
chance_points="single",
recalc=100,
risk_obj=False,
risk=0.95,
generator="de",
env="nsga",
self_adaptive=False):
test_d = setup_problem(test_case,
additive_chance=True,
risk_obj=risk_obj,
self_adaptive=self_adaptive)
pst = pyemu.Pst(os.path.join(test_d, "{0}.pst".format(test_case)))
pst.control_data.noptmax = noptmax
pst.pestpp_options["mou_population_size"] = pop_size
pst.pestpp_options["opt_risk"] = risk
pst.pestpp_options["opt_stack_size"] = stack_size
pst.pestpp_options["opt_chance_points"] = chance_points
pst.pestpp_options["panther_echo"] = False
pst.pestpp_options["mou_generator"] = generator
pst.pestpp_options["mou_env_selector"] = env
pst.pestpp_options["opt_recalc_chance_every"] = recalc
pst.write(os.path.join(test_d, "{0}.pst".format(test_case)))
master_d = test_d.replace("template","master_generator={3}_env={4}_risk={0:04.3f}_riskobj={1}_chancepoints={2}_adaptive={5}".\
format(risk,risk_obj,chance_points,generator,env,self_adaptive))
pyemu.os_utils.start_workers(test_d,
exe_path,
"{0}.pst".format(test_case),
num_workers=35,
master_dir=master_d,
worker_root=test_root,
port=port)
return master_d
def ensemble_1to1_test():
try:
import matplotlib.pyplot as plt
except:
return
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
num_reals = 100
oe1 = pyemu.ObservationEnsemble.from_gaussian_draw(pst,
num_reals=num_reals)
pst.observation_data.loc[pst.nnz_obs_names, "weight"] *= 10.0
oe2 = pyemu.ObservationEnsemble.from_gaussian_draw(pst,
num_reals=num_reals)
pst.observation_data.loc[pst.nnz_obs_names, "weight"] /= 100.0
oe_base = pyemu.ObservationEnsemble.from_gaussian_draw(pst,
num_reals=num_reals)
print(oe1.loc[:, pst.nnz_obs_names].std())
print(oe2.loc[:, pst.nnz_obs_names].std())
# pyemu.plot_utils.ensemble_res_1to1(oe1,pst,filename=os.path.join("temp","e1to1.pdf"))
#
# pyemu.plot_utils.ensemble_res_1to1({"0.5":oe1,"b":oe2},pst,filename=os.path.join("temp","e1to1.pdf"))
pyemu.plot_utils.ensemble_res_1to1({
"0.5": oe1,
"b": oe2
},
pst,
filename=os.path.join(
"temp", "e1to1_noise.pdf"),
base_ensemble=oe_base)
pyemu.plot_utils.res_phi_pie(pst=pst, ensemble=oe1)
pyemu.plot_utils.res_1to1(pst=pst, ensemble=oe1)
def ensemble_summary_test():
try:
import matplotlib.pyplot as plt
except:
return
pst = pyemu.Pst(os.path.join("pst","pest.pst"))
num_reals = 100
oe1 = pyemu.ObservationEnsemble.from_id_gaussian_draw(pst,num_reals=num_reals)
pst.observation_data.loc[pst.nnz_obs_names,"weight"] *= 10.0
oe2 = pyemu.ObservationEnsemble.from_id_gaussian_draw(pst, num_reals=num_reals)
#print(oe1.loc[:,pst.nnz_obs_names].std())
#print(oe2.loc[:,pst.nnz_obs_names].std())
pyemu.plot_utils.ensemble_change_summary(oe1,oe2,pst,filename=os.path.join("temp","edeltasum.pdf"))
pst.parameter_data.loc[:,"partrans"] = "none"
cov1 = pyemu.Cov.from_parameter_data(pst,sigma_range=6)
pe1 = pyemu.ParameterEnsemble.from_gaussian_draw(pst,cov1,num_reals=1000)
cov2 = cov1 * 0.001
pe2 = pyemu.ParameterEnsemble.from_gaussian_draw(pst,cov2,num_reals=1000)
pyemu.plot_utils.ensemble_change_summary(pe1, pe2, pst, filename=os.path.join("temp", "edeltasum.pdf"))
def obs_gauss_draw_consistency_test():
pst = pyemu.Pst(os.path.join("pst","pest.pst"))
num_reals = 1000
oe1 = pyemu.ObservationEnsemble.from_gaussian_draw(pst,num_reals=num_reals)
cov = pyemu.Cov.from_observation_data(pst).to_2d()
oe2 = pyemu.ObservationEnsemble.from_gaussian_draw(pst,cov=cov,num_reals=num_reals)
oe3 = pyemu.ObservationEnsemble.from_gaussian_draw(pst,cov=cov,num_reals=num_reals,by_groups=False)
theo_mean = pst.observation_data.obsval.copy()
#theo_mean.loc[pst.nnz_obs_names] = 0.0
theo_std = 1.0 / pst.observation_data.loc[pst.nnz_obs_names,"weight"]
for oe in [oe1,oe2,oe3]:
assert oe.shape[0] == num_reals
assert oe.shape[1] == pst.nnz_obs
d = (oe.mean() - theo_mean).apply(np.abs)
assert d.max() < 0.01,d.sort_values()
d = (oe.loc[:,pst.nnz_obs_names].std() - theo_std)
assert d.max() < 0.01,d.sort_values()
# ensemble should be transformed so now lets test the I/O
oe_org = oe.copy()
oe.to_binary("test.jcb")
oe = pyemu.ObservationEnsemble.from_binary(pst=pst, filename="test.jcb")
oe._df.index = oe.index.map(np.int)
d = (oe - oe_org).apply(np.abs)
assert d.max().max() < 1.0e-10, d.max().sort_values(ascending=False)
oe.to_csv("test.csv")
oe = pyemu.ObservationEnsemble.from_csv(pst=pst,filename="test.csv")
d = (oe - oe_org).apply(np.abs)
assert d.max().max() < 1.0e-10,d.max().sort_values(ascending=False)
def salib_verf():
import pyemu
from SALib.sample import saltelli
from SALib.analyze import sobol
m_d = os.path.join("sen_invest","master_sobol")
v_d = os.path.join("sen_invest","verf")
if os.path.exists(v_d):
shutil.rmtree(v_d)
os.makedirs(v_d)
pst = pyemu.Pst(os.path.join(m_d,"pest.pst"))
pst.add_transform_columns()
bounds = [[l,u] for l,u in zip(pst.parameter_data.parlbnd_trans,pst.parameter_data.parubnd_trans)]
problem = {"num_vars":pst.npar_adj,"names":pst.par_names,"bounds":bounds}
test = saltelli.sample(problem,100,calc_second_order=False)
out_df = pd.read_csv(os.path.join(m_d,"pest.sobol.obs.csv"),index_col=0)
reorder_df = out_df.copy()
idx = [0,3,1,2]
for i in range(4):
s = i*5
e = s + 5
chunk = out_df.iloc[s:e,:].copy()
reorder_df.iloc[idx[i]::4,:] = chunk.values
print(chunk.p1,reorder_df.iloc[idx[i]::4,:].p1)
pass
si_vals = pd.DataFrame(columns=pst.par_names,index=pst.obs_names)
sti_vals = pd.DataFrame(columns=pst.par_names, index=pst.obs_names)
for obs_name in pst.obs_names:
#if obs_name != "p1":
# continue
si = sobol.analyze(problem,reorder_df.loc[:,obs_name].values,calc_second_order=False,num_resamples=5)
print(obs_name,si)
si_vals.loc[obs_name,:] = si["S1"]
sti_vals.loc[obs_name, :] = si["ST"]
si_vals.to_csv(os.path.join(v_d,"si.csv"))
sti_vals.to_csv(os.path.join(v_d, "sti.csv"))
def add_base_test():
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
num_reals = 10
pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, num_reals=num_reals)
oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals)
pet = pe.copy()
pet.transform()
pe.add_base()
assert pe.shape[0] == num_reals
pet.add_base()
assert pet.shape[0] == num_reals
assert "base" in pe.index
assert "base" in pet.index
p = pe.loc["base", :]
d = (pst.parameter_data.parval1 - pe.loc["base", :]).apply(np.abs)
pst.add_transform_columns()
d = (pst.parameter_data.parval1_trans - pet.loc["base", :]).apply(np.abs)
assert d.max() == 0.0
try:
pe.add_base()
except:
pass
else:
raise Exception("should have failed")
oe.add_base()
assert oe.shape[0] == num_reals
d = (pst.observation_data.loc[oe.columns, "obsval"] -
oe.loc["base", :]).apply(np.abs)
assert d.max() == 0
try:
oe.add_base()
except:
pass
else:
raise Exception("should have failed")
def lorenz96_basic_test():
pst_setup_ES_new(N=50)
t_d = mod_to_seq()
# invest()
m_d = "master_da_test"
if os.path.exists(m_d):
shutil.rmtree(m_d)
pst = pyemu.Pst(os.path.join(t_d, "enkf.pst"))
pst.control_data.noptmax = 1
pst.pestpp_options["da_use_simulated_states"] = False
pst.pestpp_options["da_num_reals"] = 10
par = pst.parameter_data
istates_pars = par.loc[par.parnme.str.startswith("x"), "parnme"]
par.loc[istates_pars, "partrans"] = "fixed"
pst.write(os.path.join(t_d, "enkf.pst"), version=2)
pyemu.os_utils.start_workers(t_d,
exe_path,
"enkf.pst",
num_workers=10,
master_dir=m_d,
worker_root=".",
port=port)
# find how many cycles we are using
num_cycles = None
with open(os.path.join(m_d, "enkf.rec")) as f:
for line in f:
if line.startswith("...assimilating over"):
raw = line.strip().split()
num_cycles = int(raw[2])
assert num_cycles is not None
# find the global pe files
gpe_files = [f for f in os.listdir(m_d) if "global" in f and "pe" in f]
assert len(gpe_files) == num_cycles + 1, "{0} vs {1}".format(
num_cycles, len(gpe_files))
def alternative_dw():
import os
import pyemu
import numpy as np
import pandas as pd
wdir = os.path.join("..", "verification", "Freyberg")
sc = pyemu.Schur(os.path.join(wdir, "freyberg.jcb"))
print(sc.pst.nnz_obs)
obs = sc.pst.observation_data
zw_obs = obs.loc[obs.weight == 0, "obsnme"].iloc[:5].values
test_obs = {o: o for o in zw_obs}
base = sc.get_added_obs_importance(obslist_dict=test_obs)
zw_pst = pyemu.Pst(os.path.join(wdir, "freyberg.pst"))
zw_pst.observation_data.loc[:, "weight"] = 0.0
for o in zw_obs:
ojcb = sc.jco.get(row_names=[o])
zw_pst.observation_data.loc[:, "weight"] = 0.0
zw_pst.observation_data.loc[o, "weight"] = 1.0
sc_o = pyemu.Schur(jco=ojcb,
pst=zw_pst,
parcov=sc.posterior_parameter,
forecasts=sc.forecasts)
print(sc_o.get_forecast_summary())
def test():
pst = pyemu.Pst(os.path.join(pestpp_dir, "pest.pst"))
m = pyemu.ObservationEnsemble.from_binary(
pst, os.path.join(pestpp_dir, "restart_obs.jcb"))
print(m)
def ext_stdcol_test():
model_d = "ies_10par_xsec"
local=True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local=False
t_d = os.path.join(model_d,"template")
m_d = os.path.join(model_d,"master_ext_stdcol")
if os.path.exists(m_d):
shutil.rmtree(m_d)
shutil.copytree(t_d,m_d)
pst = pyemu.Pst(os.path.join(m_d,"pest.pst"))
obs = pst.observation_data
obs.loc[pst.nnz_obs_names,"standard_deviation"] = 1/obs.loc[pst.nnz_obs_names,"weight"]
pst.add_transform_columns()
par = pst.parameter_data
par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0
#par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"]
pst.pestpp_options["ies_num_reals"] = 10
pst.control_data.noptmax = -1
pst.write(os.path.join(m_d,"pest_base.pst"))
pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d)
pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2)
pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d)
df1 = pd.read_csv(os.path.join(m_d,"pest_base.phi.meas.csv"),index_col=0)
df2 = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.phi.meas.csv"),index_col=0)
d = (df1 - df2).apply(np.abs)
print(d.max())
assert d.max().max() < 1.0e-6,d.max().max()
pst.pestpp_options["ies_num_reals"] = 100000
pst.control_data.noptmax = -2
obs = pst.observation_data
obs.loc[pst.nnz_obs_names,"standard_deviation"] = 7.5
pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2)
pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d)
df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names]
d = (df.std() - obs.loc[pst.nnz_obs_names,"standard_deviation"]).apply(np.abs)
print(d)
assert d.max() < 0.1,d.max()
obs = pst.observation_data
obs.loc[pst.nnz_obs_names,"upper_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 1.1
obs.loc[pst.nnz_obs_names,"lower_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 0.9
par = pst.parameter_data
par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"]
pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2)
pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d)
df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names]
mn = df.min()
mx = df.max()
dmn = mn - obs.loc[pst.nnz_obs_names,"obsval"] * 0.9
print(obs.loc[pst.nnz_obs_names,"obsval"] * 0.9)
print(mn)
print(dmn)
dmx = mx - obs.loc[pst.nnz_obs_names,"obsval"] * 1.1
print(obs.loc[pst.nnz_obs_names,"obsval"] * 1.1)
print(mx)
print(dmx)
dmn = dmn.apply(np.abs)
dmx = dmx.apply(np.abs)
assert dmn.max() < 1.0e-6,dmn
assert dmx.max() < 1.0e-6,dmx
def tplins1_test():
model_d = "tplins_test_1"
t_d = os.path.join(model_d, "test")
if os.path.exists(t_d):
shutil.rmtree(t_d)
shutil.copytree(os.path.join(model_d,"template"),t_d)
pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
ins_file = os.path.join(t_d,"AOC_obs.txt.ins")
pst.add_observations(ins_file,ins_file.replace(".ins",""),pst_path=".")
pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d)
obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"])
obf_df.index = obf_df.obsnme
pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
res_df = pst.res
d = (obf_df.obsval - res_df.modelled).apply(np.abs)
#print(d)
print(d.max())
assert d.max() < 1.0e-5, d
jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs)
assert jco.sum().sum() == 0, jco.sum()
# check the input file - the last two number should be the same
arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref"))
assert arr[-2] == arr[-1]
lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines()
lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines()
assert len(lines_tpl) - 1 == len(lines_in)
pst = pyemu.Pst(os.path.join(t_d,"pest.pst"))
pst.pestpp_options["fill_tpl_zeros"] = True
pst.write(os.path.join(t_d,"pest_fill.pst"))
pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d)
obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"])
obf_df.index = obf_df.obsnme
pst = pyemu.Pst(os.path.join(t_d,"pest_fill.pst"))
res_df = pst.res
d = (obf_df.obsval - res_df.modelled).apply(np.abs)
#print(d)
print(d.max())
assert d.max() < 1.0e-5, d
jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs)
assert jco.sum().sum() == 0, jco.sum()
# check the input file - the last two number should be the same
arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref"))
assert arr[-2] == arr[-1]
lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines()
lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines()
assert len(lines_tpl) - 1 == len(lines_in)
pst = pyemu.Pst(os.path.join(t_d, "pest.pst"))
dum_obs = ['h01_03', 'h01_07']
pst.observation_data.drop(index=dum_obs, inplace=True)
pst.instruction_files = ['out1dum.dat.ins']
pst.write(os.path.join(t_d, "pest_dum.pst"))
pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d)
obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None,
names=["obsnme", "obsval"])
obf_df.index = obf_df.obsnme
pst = pyemu.Pst(os.path.join(t_d, "pest_dum.pst"))
res_df = pst.res
d = (obf_df.obsval - res_df.modelled).apply(np.abs)
# print(d)
print(d.max())
assert d.max() < 1.0e-5, d
jco = pyemu.Jco.from_binary(os.path.join(t_d, "pest_dum.jcb")).to_dataframe().apply(np.abs)
assert jco.sum().sum() == 0, jco.sum()