def run():
pst = pyemu.Pst("supply2_pest.fosm.pst")
base_par_names = pst.adj_par_names
new_par_names = [name for name in base_par_names if name[0] in ["t", "s"]]
print(new_par_names)
new_pst = pst.get(par_names=new_par_names)
forecast_names = pst.observation_data.groupby(
pst.observation_data.obgnme).groups["less_obs"]
#new_pst.observation_data.loc[forecast_names,"weight"] = 0.0
sc = pyemu.Schur(jco="supply2_pest.full.jcb",
pst=new_pst,
forecasts=forecast_names,
verbose=True)
print(sc.obscov)
#sum = sc.get_forecast_summary().loc[:,"post_var"].apply(np.sqrt)
#print(sum)
#df_worth = sc.get_added_obs_importance(reset_zero_weight=1.0)
#df_worth.to_csv("worth.csv")
df_next = sc.next_most_important_added_obs(reset_zero_weight=1.0)
df_next.to_csv("next.csv")
def mc_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
out_dir = os.path.join("mc")
if not os.path.exists(out_dir):
os.mkdir(out_dir)
#write testing
mc = MonteCarlo(jco=jco,verbose=True,sigma_range=6)
cov = Cov.from_parameter_data(mc.pst,sigma_range=6)
assert np.abs((mc.parcov.x - cov.x).sum()) == 0.0
mc.draw(10,obs=True)
mc.write_psts(os.path.join("temp","real_"))
mc.parensemble.to_parfiles(os.path.join("mc","real_"))
mc = MonteCarlo(jco=jco,verbose=True)
mc.draw(10,obs=True)
print("prior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
projected_en = mc.project_parensemble(inplace=False)
print("projected ensemble variance:",
np.var(projected_en.loc[:,"mult1"]))
import pyemu
sc = pyemu.Schur(jco=jco)
mc = MonteCarlo(pst=pst,parcov=sc.posterior_parameter,verbose=True)
mc.draw(10)
print("posterior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
def load_jco_test():
import os
import pyemu
jco = pyemu.Jco.from_binary(os.path.join("mat", "base_pest.jco"))
sc = pyemu.Schur(jco=os.path.join("mat", "base_pest.jco"),
parcov=os.path.join("mat", "parameters.unc"))
print(sc.get_parameter_summary())
def predunc7_test():
post_pd7 = pyemu.Cov.from_ascii(post_mat)
la_ord = pyemu.Schur(jco=ord_base + ".jco", predictions=predictions)
post_pyemu = la_ord.posterior_parameter
delta_sum = np.abs((post_pd7 - post_pyemu).x).sum()
print("delta matrix sum: {0:15.6E}".format(delta_sum))
assert delta_sum < 1.0e-4
def forecast_pestpp_load_test():
import os
import pyemu
pst_name = os.path.join("pst", "forecast.pst")
jco_name = pst_name.replace(".pst", ".jcb")
pst = pyemu.Pst(pst_name)
print(pst.pestpp_options)
sc = pyemu.Schur(jco=jco_name)
print(sc.get_forecast_summary())
def std_weights_test():
d = os.path.join("opt_dewater_chance", "test_std_weights2")
if os.path.exists(d):
shutil.rmtree(d)
shutil.copytree(os.path.join("opt_dewater_chance", "template"), d)
pst_file = os.path.join(d, "dewater_pest.base.pst")
jco_file = os.path.join(d, "dewater_pest.full.jcb")
pst = pyemu.Pst(pst_file)
par = pst.parameter_data
par.loc[par.partrans == "fixed", "partrans"] = "log"
jco = pyemu.Jco.from_binary(jco_file)
par.loc[par.pargp == "q", "partrans"] = "fixed"
obs = pst.observation_data.loc[pst.nnz_obs_names, :]
forecast_names = list(obs.loc[obs.obgnme.apply(lambda x: x.startswith("l_") or \
x.startswith("less_")), "obsnme"])
# print(forecast_names)
pst.observation_data.loc[:, "weight"] = 0.0
sc = pyemu.Schur(jco=jco, pst=pst, forecasts=forecast_names)
# print(sc.get_forecast_summary())
fstd = sc.get_forecast_summary().loc[:, "post_var"].apply(np.sqrt)
pr_unc_py = fstd.to_dict()
pst.observation_data.loc[fstd.index, "weight"] = fstd.values
pst.pestpp_options["opt_risk"] = 0.1
pst.pestpp_options["opt_std_weights"] = True
pst.pestpp_options["base_jacobian"] = os.path.split(jco_file)[-1]
par.loc[par.pargp == "q", "partrans"] = "none"
new_pst_file = os.path.join(d, "test.pst")
pst.write(new_pst_file)
pyemu.os_utils.run("{0} {1}".format(exe_path,
os.path.split(new_pst_file)[-1]),
cwd=d)
pr_unc1 = scrap_rec(new_pst_file.replace(".pst", ".rec"))
print(pr_unc1)
for fore in forecast_names:
dif = np.abs(pr_unc_py[fore] - pr_unc1[fore])
print(fore, pr_unc_py[fore], pr_unc1[fore], dif)
assert dif < 1.0e-4
pst.pestpp_options["opt_std_weights"] = False
pst.write(new_pst_file)
pyemu.os_utils.run("{0} {1}".format(exe_path,
os.path.split(new_pst_file)[-1]),
cwd=d)
pr_unc2 = scrap_rec(new_pst_file.replace(".pst", ".rec"))
print(pr_unc2)
for fore in forecast_names:
dif = np.abs(pr_unc_py[fore] - pr_unc2[fore])
print(fore, pr_unc_py[fore], pr_unc2[fore], dif)
assert dif < 1.0e-4
def alternative_dw():
import os
import pyemu
import numpy as np
import pandas as pd
wdir = os.path.join("..","verification","Freyberg")
sc = pyemu.Schur(os.path.join(wdir,"freyberg.jcb"))
print(sc.pst.nnz_obs)
obs = sc.pst.observation_data
zw_obs = obs.loc[obs.weight==0,"obsnme"].iloc[:5].values
test_obs = {o:o for o in zw_obs}
base = sc.get_added_obs_importance(obslist_dict=test_obs)
zw_pst = pyemu.Pst(os.path.join(wdir,"freyberg.pst"))
zw_pst.observation_data.loc[:,"weight"] = 0.0
for o in zw_obs:
ojcb = sc.jco.get(row_names=[o])
zw_pst.observation_data.loc[:,"weight"] = 0.0
zw_pst.observation_data.loc[o,"weight"] = 1.0
sc_o = pyemu.Schur(jco=ojcb,pst=zw_pst,parcov=sc.posterior_parameter,forecasts=sc.forecasts)
print(sc_o.get_forecast_summary())
def predunc1_test():
la_ord = pyemu.Schur(jco=ord_base+".jco",predictions=predictions)
fsum = la_ord.get_forecast_summary()
fsum.loc[:,["prior_var","post_var"]] = fsum.loc[:,["prior_var","post_var"]].apply(np.sqrt)
# load the predunc1 results
pd1_results = pd.read_csv(os.path.join(verf_dir,"predunc1_results.dat"))
pd1_results.index = ["prior_var","post_var"]
for forecast_name in fsum.index:
pd1_pr,pd1_pt = pd1_results.loc[:,forecast_name]
pr,pt = fsum.loc[forecast_name,["prior_var","post_var"]].values
pr_diff = np.abs(pr - pd1_pr)
pt_diff = np.abs(pt - pd1_pt)
print("forecast:",forecast_name,"prior diff:{0:15.6E}".format(pr_diff),\
"post diff:{0:15.6E}".format(pt_diff))
assert pr_diff < 1.0e-3
assert pt_diff < 1.0e-3
def par_contrib_speed_test():
import os
import numpy as np
import pyemu
npar = 1800
nobs = 1000
nfore = 1000
par_names = ["par{0}".format(i) for i in range(npar)]
obs_names = ["obs{0}".format(i) for i in range(nobs)]
fore_names = ["fore{0}".format(i) for i in range(nfore)]
all_names = copy.deepcopy(obs_names)
all_names.extend(fore_names)
pst = pyemu.Pst.from_par_obs_names(par_names, all_names)
cal_jco = pyemu.Jco.from_names(obs_names, par_names, random=True)
fore_jco = pyemu.Jco.from_names(par_names, fore_names, random=True)
pst.observation_data.loc[obs_names, "weight"] = 1.0
pst.observation_data.loc[fore_names, "weight"] = 0.0
sc = pyemu.Schur(jco=cal_jco, pst=pst, forecasts=fore_jco, verbose=True)
sc.get_par_contribution(parlist_dict={par_names[0]: par_names[0]})
import numpy as np
import pyemu
pst = pyemu.Pst("supply2_pest.sfr1.fromfulljco.pst")
obs = pst.observation_data
pst.prior_information = pst.null_prior
forecasts = obs.groupby(obs.obgnme).groups["less_obs"]
pars = [par for par in pst.adj_par_names if not par[0] in ["i",'q']]
obs = [obs for obs in pst.nnz_obs_names if obs not in forecasts]
print(pars,obs)
sc = pyemu.Schur(jco="supply2_pest.full.jcb",pst=pst,forecasts=forecasts)
sc = sc.get(par_names=pars,obs_names=obs)
print(sc.pst.npar_adj,sc.pst.nnz_obs)
df = sc.get_forecast_summary().loc[:,["prior_var","post_var"]].apply(np.sqrt)
print(df)
sc.posterior_parameter.to_ascii("supply2_pest.fosm.post.cov")
pst = pyemu.Pst("supply2_pest.sfr1.fromfulljco.pst")
obs = pst.observation_data
names = obs.groupby(obs.obgnme.apply(lambda x:x=="less_obs")).groups[False]
obs.loc[names,"weight"] = 0.0
pst.pestpp_options["parcov_filename"] = "supply2_pest.fosm.post.cov"
pst.write("supply2_pest.sfr1.fromfulljco.postcov.pst")
def freyberg_verf_test():
import os
import pyemu
import numpy as np
import pandas as pd
wdir = os.path.join("..", "verification", "Freyberg")
post_pd7 = pyemu.Cov.from_ascii(os.path.join(wdir, "post.cov"))
sc = pyemu.Schur(os.path.join(wdir, "freyberg.jcb"))
post_pyemu = sc.posterior_parameter
diff = (post_pd7 - post_pyemu).to_dataframe()
diff = (diff / sc.pst.parameter_data.parval1 * 100.0).apply(np.abs)
print(diff.max().max())
assert diff.max().max() < 10.0
pd1_file = os.path.join(wdir, "predunc1_textable.dat")
names = ["forecasts", "pd1_pr", "py_pr", "pd1_pt", "py_pt"]
pd1_df = pd.read_csv(pd1_file, sep='&', header=None, names=names)
pd1_df.index = pd1_df.forecasts
fsum = sc.get_forecast_summary()
pd1_cols = ["pd1_pr", "pd1_pt"]
py_cols = ["prior_var", "post_var"]
forecasts = ["sw_gw_0", "sw_gw_1"]
for fname in forecasts:
for pd1_col, py_col in zip(pd1_cols, py_cols):
pd1_pr = pd1_df.loc[fname, pd1_col]
py_pr = np.sqrt(fsum.loc[fname, py_col])
assert np.abs(pd1_pr - py_pr) < 1.0e-1, "{0},{1}".format(
pd1_pr, py_pr)
out_files = [os.path.join(wdir,f) for f in os.listdir(wdir)\
if ".predvar1b.out" in f and f.split('.')[0] in forecasts]
print(out_files)
pv1b_results = {}
for out_file in out_files:
pred_name = os.path.split(out_file)[-1].split('.')[0]
f = open(out_file, 'r')
for _ in range(3):
f.readline()
arr = np.loadtxt(f)
pv1b_results[pred_name] = arr
omitted_parameters = [pname for pname in sc.pst.parameter_data.parnme if\
pname.startswith("wf")]
obs_names = sc.pst.nnz_obs_names
obs_names.extend(forecasts)
jco = pyemu.Jco.from_binary(os.path.join(wdir,"freyberg.jcb")).\
get(obs_names,sc.pst.adj_par_names)
ev = pyemu.ErrVar(jco=jco,
pst=sc.pst,
forecasts=forecasts,
omitted_parameters=omitted_parameters,
verbose=False)
#print(ev.jco.shape)
df = ev.get_errvar_dataframe(np.arange(36))
#print(df)
max_idx = 12
#print(pv1b_results.keys())
for ipred, pred in enumerate(forecasts):
arr = pv1b_results[pred][:max_idx, :]
first = df[("first", pred)][:max_idx]
second = df[("second", pred)][:max_idx]
third = df[("third", pred)][:max_idx]
#print(arr[:,1])
#print(first)
diff = np.abs(arr[:, 1] - first) / arr[:, 1] * 100.0
assert diff.max() < 0.01
diff = np.abs(arr[:, 2] - second) / arr[:, 1] * 100.0
assert diff.max() < 0.01
diff = np.abs(arr[:, 3] - third) / arr[:, 1] * 100.0
assert diff.max() < 0.01
# In[2]:
# the path to where the jacobian matrix is in the file system
jco_path = os.path.join("..","FreybergModelForClass","Freyberg_K_only","freyberg.jcb")
# and path to the pest control file
pst_path = jco_path.replace(".jcb",".pst")
# In[4]:
# load the control file
pst = pyemu.Pst(pst_path)
# get the forecast names
forecast_names = pst.pestpp_options["forecasts"].split(',')
sc = pyemu.Schur(jco=jco_path)#,forecasts=forecast_names)
# In[5]:
sc.get_parameter_summary()
# In[6]:
sc.get_forecast_summary()
# In[ ]:
#---- Load the jacobian matrix to a dataframe
jco_mat = pyemu.Jco.from_binary(jco) # Load jacobian matrix
jco_df = jco_mat.to_dataframe() # Write to dataframe
#---- List of model forecasts
forecasts = jco_df[jco_df.index.str.startswith(
('zmuni', 'fw_storage'))].index.to_list(
) # All observations starting with 'zmuni' or 'fw_storage'
#---- Create linear analysis object
la = pyemu.Schur(jco=jco,
pst=pst_bpa,
parcov=None,
obscov=None,
forecasts=forecasts,
sigma_range=4.0)
print('# observations: ' + str(la.jco.shape[0]) +
', # adjustable parameters: ' + str(la.jco.shape[1]) +
', # forecasts: ' + str(len(la.forecast_names)))
#%%---------------- Prior vs posterior forecast distributions -----------------
# Retrieve prior & posterior forecast uncertainty (as variances)
for_sum = la.get_forecast_summary(
) # Dataframe containing prior & posterior forecasts variances (and percent reduction in variance)
# Compute the standard deviation from the variance
import numpy as np
import pyemu
pst = pyemu.Pst("supply2_pest.pst")
base_par_names = pst.adj_par_names
new_par_names = [name for name in base_par_names if name[0] in ["t", "s"]]
print(new_par_names)
new_pst = pst.get(par_names=new_par_names)
forecast_names = pst.observation_data.groupby(
pst.observation_data.obgnme).groups["less_obs"]
sc = pyemu.Schur(jco="supply2_pest.3.jcb",
pst=new_pst,
forecasts=forecast_names)
sum = sc.get_forecast_summary().loc[:, "post_var"].apply(np.sqrt)
print(sum)
import numpy as np
import pyemu
pst = pyemu.Pst("supply2_pest.fosm.pst")
base_par_names = pst.adj_par_names
new_par_names = [name for name in base_par_names if name[0] in ["t","s"]]
print(new_par_names)
new_pst = pst.get(par_names=new_par_names)
forecast_names = pst.observation_data.groupby(pst.observation_data.obgnme).groups["less_obs"]
#new_pst.observation_data.loc[forecast_names,"weight"] = 0.0
sc = pyemu.Schur(jco="supply2_pest.full.jcb",pst=new_pst,forecasts=forecast_names,verbose=True)
print(sc.obscov)
#sum = sc.get_forecast_summary().loc[:,"post_var"].apply(np.sqrt)
#print(sum)
df_worth = sc.get_added_obs_importance(reset_zero_weight=1.0)
df_worth.to_csv("worth.csv")
new_obs_names = ["test1", "test2"]
new_df = pd.DataFrame(
{
"obsnme": new_obs_names,
"weight": 0.0,
"obgnme": "swgwex",
"obsval": -999.0
},
index=new_obs_names)
pst = pyemu.Pst("base_pest.pst")
pst.observation_data = pst.observation_data.append(new_df)
pst.write("test.pst")
exit()
jco1 = pyemu.Jco.from_binary("base_pest.jco")
sc = pyemu.Schur(pst="base_pest.pst", jco=jco1, verbose=True)
par_sum = sc.get_parameter_summary()
par_sum.sort_values(by="percent_reduction", inplace=True)
print(par_sum)
sc.get_par_group_contribution()
dw_df = sc.get_removed_obs_importance()
# la = pyemu.ErrVar(pst="base_pest.pst",jco=jco1)
# css_df = la.get_par_css_dataframe()
# css_df.sort_values(by="pest_css",inplace=True,ascending=False)
# print(css_df.pest_css)
#css_df.pest_css.iloc[:10].plot(kind="bar")
#plt.show()
# ident_df = la.get_identifiability_dataframe(20)
# print(ident_df.columns)
import numpy as np
import pyemu
pst = pyemu.Pst("restart_org.pst")
jco = pyemu.Jco.from_binary("restart_org.jcb")
# get the less than constraints and set to zero weight
obs = pst.observation_data
forecast_names = list(obs.loc[
obs.apply(lambda x: x.weight > 0 and x.obgnme.startswith("less_"), axis=1),
"obsnme"].values)
obs.loc[forecast_names, "weight"] = 0.0
sc = pyemu.Schur(pst=pst, jco=jco, forecasts=forecast_names)
prior_fore = sc.prior_forecast.items()
# reload since the sc extract the forecast rows
pst = pyemu.Pst("restart_org.pst")
jco = pyemu.Jco.from_binary("restart_org.jcb")
rei = pyemu.pst_utils.read_resfile("restart_org.rei")
for n, v in prior_fore:
pst.observation_data.loc[n, "weight"] = np.sqrt(v)
#sqrt then set weight
keep_obs = pst.nnz_obs_names
keep_pars = list(pst.parameter_data.loc[pst.parameter_data.pargp == "k1",
"parnme"].values)
pst.observation_data = pst.observation_data.loc[keep_obs, :]
pst.parameter_data = pst.parameter_data.loc[keep_pars, :]