def enforce_test():
import os
import pyemu
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"),
verbose=True)
cov = pyemu.Cov(x=mc.parcov.x * 0.1,
names=mc.parcov.row_names,
isdiagonal=True)
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"),
parcov=cov)
mc.draw(num_reals=100, enforce_bounds='drop')
assert mc.parensemble.shape[0] == 100.0
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"))
mc.draw(num_reals=100, enforce_bounds='drop')
assert mc.parensemble.shape[0] == 0
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"))
mc.draw(100, enforce_bounds="reset")
diff = mc.parensemble.ubnd - mc.parensemble.max(axis=0)
assert diff.min() == 0.0
diff = mc.parensemble.lbnd - mc.parensemble.min(axis=0)
assert diff.max() == 0.0
def enforce_test():
import os
import pyemu
dir = "mc"
mc = pyemu.MonteCarlo(jco=os.path.join("mc","freyberg_ord.jco"))
mc.draw(num_reals=100,enforce_bounds='drop')
assert mc.parensemble.shape[0] == 0
cov = pyemu.Cov(x=mc.parcov.x * 0.1,names=mc.parcov.row_names,isdiagonal=True)
mc = pyemu.MonteCarlo(jco=os.path.join("mc","freyberg_ord.jco"),
parcov=cov)
mc.draw(num_reals=100,enforce_bounds='drop')
assert mc.parensemble.shape[0] == 100.0
def ensemble_covariance_test():
import os
import numpy as np
import pyemu
from datetime import datetime
v = pyemu.geostats.ExpVario(contribution=1.0, a=1.0)
gs = pyemu.geostats.GeoStruct(variograms=[v])
npar = 10
pst = pyemu.pst_utils.generic_pst(["p{0:010d}".format(i) for i in range(npar)], ["o1"])
pst.parameter_data.loc[:, "partrans"] = "none"
par = pst.parameter_data
par.loc[:, "x"] = np.random.random(npar) * 10.0
par.loc[:, "y"] = np.random.random(npar) * 10.0
cov = gs.covariance_matrix(par.x, par.y, par.parnme)
num_reals = 100000
mc = pyemu.MonteCarlo(pst=pst)
peh = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov, num_reals=num_reals,use_homegrown=True)
localizer = np.ones_like(cov.x)
localizer[cov.x<1.0e-1] = 0.0
cov = cov.hadamard_product(localizer)
ecov = peh.covariance_matrix(localizer=localizer)
d = 100.0 * (np.abs((cov - ecov).x) / cov.x)
d[localizer==0.0] = np.NaN
assert np.nanmax(d) < 10.0
def pnulpar_test():
import os
import pyemu
dir = "mc"
mc = pyemu.MonteCarlo(jco=os.path.join("mc","freyberg_ord.jco"))
par_dir = os.path.join("mc","prior_par_draws")
par_files = [os.path.join(par_dir,f) for f in os.listdir(par_dir) if f.endswith('.par')]
#mc.parensemble.read_parfiles(par_files)
mc.parensemble = pyemu.ParameterEnsemble.from_parfiles(pst=mc.pst,parfile_names=par_files)
real_num = [int(os.path.split(f)[-1].split('.')[0].split('_')[1]) for f in par_files]
mc.parensemble.index = real_num
#print(mc.parensemble)
print(mc.parensemble.istransformed)
en = mc.project_parensemble(nsing=1,inplace=False, enforce_bounds='reset')
#en.index = [i+1 for i in en.index]
print(mc.parensemble.istransformed)
par_dir = os.path.join("mc", "proj_par_draws")
par_files = [os.path.join(par_dir, f) for f in os.listdir(par_dir) if f.endswith('.par')]
real_num = [int(os.path.split(f)[-1].split('.')[0].split('_')[1]) for f in par_files]
en_pnul = pyemu.ParameterEnsemble.from_parfiles(pst=mc.pst,parfile_names=par_files)
#en_pnul.read_parfiles(par_files)
en_pnul.index = real_num
en.sort_index(axis=1, inplace=True)
en.sort_index(axis=0, inplace=True)
en_pnul.sort_index(axis=1, inplace=True)
en_pnul.sort_index(axis=0, inplace=True)
diff = 100.0 * ((en - en_pnul) / en)
assert max(diff.max()) < 1.0e-4
def pnulpar_test():
pst = pyemu.Pst(ord_base + ".pst")
# load the pnulpar projected ensemble
d = os.path.join(verf_dir, "proj_par_draws")
par_files = [
os.path.join(d, f) for f in os.listdir(d) if f.startswith("draw_")
]
pnul_en = pyemu.ParameterEnsemble.from_parfiles(pst=pst,
parfile_names=par_files)
#pnul_en.read_parfiles_prefix(os.path.join(verf_dir,"proj_par_draws","draw_"))
pnul_en.loc[:, "fname"] = pnul_en.index
#pnul_en.index = pnul_en.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = pnul_en.pop("fname")
mc = pyemu.MonteCarlo(jco=ord_base + ".jco")
d = os.path.join(verf_dir, "prior_par_draws")
par_files = [
os.path.join(d, f) for f in os.listdir(d) if f.startswith("draw_")
]
#mc.parensemble.read_parfiles_prefix(os.path.join(verf_dir,"prior_par_draws","draw_"))
mc.parensemble = pyemu.ParameterEnsemble.from_parfiles(
pst=mc.pst, parfile_names=par_files)
mc.parensemble.loc[:, "fname"] = mc.parensemble.index
#mc.parensemble.index = mc.parensemble.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = mc.parensemble.pop("fname")
en = mc.project_parensemble(nsing=1, inplace=False)
diff = 100 * (np.abs(pnul_en - en) / en)
assert max(diff.max()) < 1.0e-3
def tied_test():
import os
import pyemu
pst_dir = os.path.join('..', 'tests', "pst")
pst = pyemu.Pst(os.path.join(pst_dir, "br_opt_no_zero_weighted.pst"))
print(pst.tied_lines)
pst.write(os.path.join(pst_dir, "pest_tied_tester_1.pst"))
mc = pyemu.MonteCarlo(pst=pst)
mc.draw(1)
mc.write_psts(os.path.join(pst_dir, "tiedtest_"))
def phi_vector_test():
import os
import pyemu
jco = os.path.join("pst", "pest.jcb")
pst = pyemu.Pst(jco.replace(".jcb", ".pst"))
mc = pyemu.MonteCarlo(pst=pst)
num_reals = 10
mc.draw(num_reals, obs=True)
print(mc.obsensemble.phi_vector)
def gen_fails():
mc = pyemu.MonteCarlo(pst=pst)
mc.draw(5, obs=True)
mc.parensemble.iloc[0, :] *= -10000000.0
mc.parensemble.to_csv("par.csv")
mc.obsensemble.to_csv("obs.csv")
os.system("sweep.exe {0}".format(pst_file))
df = pd.read_csv("sweep_out.csv")
print(df.loc[:, pst.observation_data.obsnme.apply(str.upper)])
def homegrown_draw_test():
import os
import numpy as np
import pyemu
from datetime import datetime
v = pyemu.geostats.ExpVario(contribution=1.0, a=1.0)
gs = pyemu.geostats.GeoStruct(variograms=[v])
npar = 1000
pst = pyemu.pst_utils.generic_pst(
["p{0:010d}".format(i) for i in range(npar)], ["o1"])
pst.parameter_data.loc[:, "partrans"] = "none"
par = pst.parameter_data
par.loc[:, "x"] = np.random.random(npar) * 10.0
par.loc[:, "y"] = np.random.random(npar) * 10.0
cov = gs.covariance_matrix(par.x, par.y, par.parnme)
num_reals = 1000
mc = pyemu.MonteCarlo(pst=pst)
s = datetime.now()
#print(s)
pe = pyemu.ParameterEnsemble.from_gaussian_draw(mc.parensemble,
cov,
num_reals=num_reals)
mc.draw(num_reals=num_reals, cov=cov)
pe = mc.parensemble
d1 = (datetime.now() - s).total_seconds()
#print(d1)
s = datetime.now()
#print(s)
peh = pyemu.ParameterEnsemble.from_gaussian_draw_homegrown(
mc.parensemble, cov, num_reals=num_reals)
d2 = (datetime.now() - s).total_seconds()
#print(d2)
#import matplotlib.pyplot as plt
for pname in peh.names:
#ax = plt.subplot(111)
m1 = pe.loc[:, pname].mean()
m2 = peh.loc[:, pname].mean()
print(par.loc[pname, "parval1"], m2, m1)
#pe.loc[:,pname].hist(ax=ax,bins=10,alpha=0.5)
#peh.loc[:,pname].hist(ax=ax,bins=10,alpha=0.5)
#plt.show()
#break
print(d2, d1)
def pnulpar_test():
pst = pyemu.Pst(ord_base+".pst")
# load the pnulpar projected ensemble
pnul_en = pyemu.ParameterEnsemble(pst)
pnul_en.read_parfiles_prefix(os.path.join(verf_dir,"proj_par_draws","draw_"))
pnul_en.loc[:,"fname"] = pnul_en.index
pnul_en.index = pnul_en.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = pnul_en.pop("fname")
mc = pyemu.MonteCarlo(jco=ord_base+".jco")
mc.parensemble.read_parfiles_prefix(os.path.join(verf_dir,"prior_par_draws","draw_"))
mc.parensemble.loc[:,"fname"] = mc.parensemble.index
mc.parensemble.index = mc.parensemble.fname.apply(lambda x:str(int(x.split('.')[0].split('_')[-1])))
f = mc.parensemble.pop("fname")
en = mc.project_parensemble(nsing=1,inplace=False)
diff = 100 * (np.abs(pnul_en - en) / en)
assert max(diff.max()) < 1.0e-3
def to_from_binary_test():
import os
import numpy as np
import pyemu
from datetime import datetime
v = pyemu.geostats.ExpVario(contribution=1.0, a=1.0)
gs = pyemu.geostats.GeoStruct(variograms=[v])
npar = 1000
pst = pyemu.pst_utils.generic_pst(
["p{0:010d}".format(i) for i in range(npar)], ["o1"])
pst.parameter_data.loc[:, "partrans"] = "none"
par = pst.parameter_data
par.loc[:, "x"] = np.random.random(npar) * 10.0
par.loc[:, "y"] = np.random.random(npar) * 10.0
cov = gs.covariance_matrix(par.x, par.y, par.parnme)
num_reals = 1000
mc = pyemu.MonteCarlo(pst=pst)
pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst,
cov,
num_reals=num_reals,
use_homegrown=True)
oe = pyemu.ObservationEnsemble.from_id_gaussian_draw(pst,
num_reals=num_reals)
pe_name = os.path.join("temp", "pe.jcb")
oe_name = os.path.join("temp", "oe.jcb")
pe.to_binary(pe_name)
oe.to_binary(oe_name)
pe1 = pyemu.ParameterEnsemble.from_binary(mc.pst, pe_name)
oe1 = pyemu.ObservationEnsemble.from_binary(mc.pst, oe_name)
pe1.index = pe1.index.map(np.int)
oe1.index = oe1.index.map(np.int)
d = (oe - oe1).apply(np.abs)
assert d.max().max() == 0.0
d = (pe - pe1).apply(np.abs)
assert d.max().max() == 0.0, d
def tied_test():
import os
import pyemu
pst_dir = os.path.join("pst")
pst = pyemu.Pst(os.path.join(pst_dir, "br_opt_no_zero_weighted.pst"))
print(pst.tied)
pst.write(os.path.join("temp", "pest_tied_tester_1.pst"))
mc = pyemu.MonteCarlo(pst=pst)
mc.draw(1)
mc.write_psts(os.path.join("temp", "tiedtest_"))
par = pst.parameter_data
par.loc[pst.par_names[::3], "partrans"] = "tied"
try:
pst.write(os.path.join("temp", "pest_tied_tester_1.pst"))
except:
pass
else:
raise Exception()
par.loc[pst.par_names[::3], "partied"] = pst.par_names[0]
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
print(pst.tied)
par = pst.parameter_data
par.loc[pst.par_names[2], "partrans"] = "tied"
print(pst.tied)
par.loc[pst.par_names[2], "partied"] = "junk1"
pst.write(os.path.join("temp", "test.pst"))
pst = pyemu.Pst(os.path.join("temp", "test.pst"))
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
print(pst.tied)
par = pst.parameter_data
par.loc[pst.par_names[2], "partrans"] = "tied"
par.loc[pst.par_names[2], "partied"] = "junk"
pst.write(os.path.join("temp", "test.pst"))
pst = pyemu.Pst(os.path.join("temp", "test.pst"))
def run_sweep_test():
import shutil
import numpy as np
import pandas as pd
try:
import flopy
except Exception as e:
return
import pyemu
ext = ''
bin_path = os.path.join("..", "..", "bin")
if "linux" in platform.platform().lower():
bin_path = os.path.join(bin_path, "linux")
elif "darwin" in platform.platform().lower():
bin_path = os.path.join(bin_path, "mac")
else:
bin_path = os.path.join(bin_path, "win")
ext = '.exe'
org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update")
nam_file = "freyberg.nam"
m = flopy.modflow.Modflow.load(nam_file,
model_ws=org_model_ws,
check=False,
forgive=False,
exe_name=mf_exe_name)
org_model_ws = "temp"
m.change_model_ws(org_model_ws)
m.write_input()
print("{0} {1}".format(mf_exe_name, m.name + ".nam"), org_model_ws)
pyemu.os_utils.run("{0} {1}".format(mf_exe_name, m.name + ".nam"),
cwd=org_model_ws)
hds_file = "freyberg.hds"
list_file = "freyberg.list"
for f in [hds_file, list_file]:
assert os.path.exists(os.path.join(org_model_ws, f))
new_model_ws = "template1"
props = [["upw.hk", None], ["upw.vka", None], ["upw.ss", None],
["rch.rech", None]]
hds_kperk = [[kper, 0] for kper in range(m.nper)]
temp_bc_props = [["wel.flux", kper] for kper in range(m.nper)]
spat_bc_props = [["wel.flux", 2]]
ph = pyemu.helpers.PstFromFlopyModel(nam_file,
new_model_ws,
org_model_ws,
const_props=props,
zone_props=props,
kl_props=props,
pp_props=props,
grid_props=props,
hds_kperk=hds_kperk,
sfr_pars=True,
sfr_obs=True,
spatial_bc_props=spat_bc_props,
temporal_bc_props=temp_bc_props,
remove_existing=True,
model_exe_name="mfnwt")
tmp = mf_exe_name.split(os.sep)
tmp = os.path.join(*tmp[1:]) + ext
assert os.path.exists(tmp), tmp
shutil.copy2(tmp, os.path.join(new_model_ws, "mfnwt" + ext))
pe = ph.draw(20)
bd = os.getcwd()
try:
oe = pe.run(slave_dir=new_model_ws)
print(oe.shape)
except Exception as e:
os.chdir(bd)
raise Exception(str(e))
mc = pyemu.MonteCarlo(pst=ph.pst)
mc.draw(20)
try:
mc.run(slave_dir=new_model_ws)
except Exception as e:
os.chdir(bd)
raise Exception(str(e))
assert mc.obsensemble.shape == oe.shape
# In[1]: #get_ipython().magic(u'matplotlib inline') import os import shutil import numpy as np import matplotlib.pyplot as plt import pandas as pd import pyemu # instaniate ```pyemu``` object and drop prior info. Then reorder the jacobian and save as binary. This is needed because the pest utilities require strict order between the control file and jacobian # In[2]: mc = pyemu.MonteCarlo(jco="freyberg.jcb", verbose=False) mc.drop_prior_information() jco_ord = mc.jco.get(mc.pst.obs_names, mc.pst.par_names) ord_base = "freyberg_ord" jco_ord.to_binary(ord_base + ".jco") mc.pst.control_data.parsaverun = ' ' mc.pst.write(ord_base + ".pst") # Draw some vectors from the prior and write the vectors to par files # In[3]: # setup the dirs to hold all this stuff par_dir = "prior_par_draws" proj_dir = "proj_par_draws" parfile_base = os.path.join(par_dir, "draw_")
def plot_flopy_par_ensemble_test():
import shutil
import numpy as np
try:
import flopy
except:
return
try:
import matplotlib.pyplot as plt
except:
print("error importing pyplot")
return
import pyemu
org_model_ws = os.path.join("..", "examples", "Freyberg_transient")
nam_file = "freyberg.nam"
new_model_ws = "temp_pst_from_flopy"
pp_props = [["upw.hk", 0], ["upw.hk", 1]]
helper = pyemu.helpers.PstFromFlopyModel(nam_file, new_model_ws, org_model_ws,
grid_props=pp_props, remove_existing=True,
model_exe_name="mfnwt")
pst = pyemu.Pst(os.path.join(new_model_ws,"freyberg.pst"))
mc = pyemu.MonteCarlo(pst=pst)
os.chdir(new_model_ws)
cov = pyemu.Cov.from_ascii("freyberg.pst.prior.cov")
mc.draw(100,cov=cov)
#pyemu.helpers.plot_flopy_par_ensemble(mc.pst, mc.parensemble, num_reals=None, model=helper.m)
#pyemu.helpers.plot_flopy_par_ensemble(mc.pst, mc.parensemble, num_reals=None)
#3try:
import cartopy.crs as ccrs
import cartopy.io.img_tiles as cimgt
import pyproj
# except:
# return
stamen_terrain = cimgt.StamenTerrain()
zoom = 10
def fig_ax_gen():
fig = plt.figure(figsize=(20,20))
nrow,ncol = 5,4
axes = []
for i in range(nrow*ncol):
#print(i)
ax = plt.subplot(nrow,ncol,i+1,projection=stamen_terrain.crs)
ax.set_extent([-97.775, -97.625, 30.2, 30.35])
#ax.set_extent([175.2, 176.2, -37, -38.2])
ax.add_image(stamen_terrain,zoom)
#plt.show()
axes.append(ax)
#break
return fig, axes
#fig,axes = fig_ax_gen()
#plt.show()
#return
pcolormesh_trans = ccrs.UTM(zone=14)
pyemu.helpers.plot_flopy_par_ensemble(mc.pst, mc.parensemble, num_reals=None,fig_axes_generator=fig_ax_gen,
pcolormesh_transform=pcolormesh_trans,model="freyberg.nam")
os.chdir("..")