def extend_test():
import numpy as np
import pyemu
first = pyemu.Cov(x=np.ones((3,3))+1.0,names=["p1","p2","p3"],isdiagonal=False)
second = pyemu.Cov(x=np.ones((4,1))+2.0,names=["o1","o2","o3","o4"],isdiagonal=True)
third = first.extend(second)
print(third)
assert third.x[:,0].sum() == 6
assert third.x[0,:].sum() == 6
assert third.x[:,2].sum() == 6
assert third.x[2,:].sum() == 6
assert third.x[:,3].sum() == 3
assert third.x[3,:].sum() == 3
assert third.x[:,6].sum() == 3
assert third.x[6,:].sum() == 3
try:
forth = pyemu.mat.concat([first,third])
except:
pass
else:
raise Exception()
forth = pyemu.Matrix(x=first.x,row_names=first.row_names,col_names=[str(i) for i in range(first.shape[1])])
x = pyemu.mat.concat([first,forth])
print(x)
fifth = pyemu.Matrix(x=first.x, row_names=[str(i) for i in range(first.shape[0])], col_names=first.col_names)
x = pyemu.mat.concat([first,fifth])
print(x)
def chenoliver():
import os
import numpy as np
import pyemu
os.chdir(os.path.join("smoother", "chenoliver"))
csv_files = [
f for f in os.listdir('.') if f.endswith(".csv") and "bak" not in f
]
[os.remove(csv_file) for csv_file in csv_files]
parcov = pyemu.Cov(x=np.ones((1, 1)), names=["par"], isdiagonal=True)
pst = pyemu.Pst("chenoliver.pst")
obscov = pyemu.Cov(x=np.ones((1, 1)) * 16.0,
names=["obs"],
isdiagonal=True)
es = pyemu.EnsembleSmoother(pst,
parcov=parcov,
obscov=obscov,
num_slaves=20,
use_approx=False)
es.initialize(num_reals=100)
for it in range(40):
es.update()
os.chdir(os.path.join("..", ".."))
def cov_replace_test():
import os
import numpy as np
import pyemu
pst = pyemu.Pst(os.path.join("pst", "pest.pst"))
cov1 = pyemu.Cov.from_parameter_data(pst)
cov2 = pyemu.Cov(x=cov1.x[:3], names=cov1.names[:3], isdiagonal=True) * 3
cov1.replace(cov2)
assert cov1.x[0] == cov2.x[0]
cov2 = pyemu.Cov(x=np.ones(cov1.shape) * 2, names=cov1.names[::-1])
cov2 = cov2.get(row_names=cov2.names[:3], col_names=cov2.names[:3])
cov1.replace(cov2)
assert cov1.x[-1, -1] == 2.0
def hadamard_product_test():
import os
import numpy as np
import pyemu
jco = pyemu.Jco.from_binary(os.path.join("mat", "pest.jcb"))
z = jco.zero2d
#print(jco.shape)
#print(z.shape)
hp = jco.hadamard_product(z)
assert hp.x.sum() == 0.0
hp = z.hadamard_product(hp)
assert hp.x.sum() == 0.0
c = pyemu.Cov(x=np.ones((jco.shape[0], 1)),
names=jco.row_names,
isdiagonal=True)
r = pyemu.Matrix(x=np.random.rand(c.shape[0], c.shape[0]),
row_names=c.row_names,
col_names=c.col_names)
hp = c.hadamard_product(r)
assert hp.x.sum() == np.diagonal(r.x).sum()
hp = r.hadamard_product(c)
assert hp.x.sum() == np.diagonal(r.x).sum()
def extend_test():
import numpy as np
import pyemu
first = pyemu.Cov(x=np.ones((3,3))+1.0,names=["p1","p2","p3"],isdiagonal=False)
second = pyemu.Cov(x=np.ones((4,1))+2.0,names=["o1","o2","o3","o4"],isdiagonal=True)
third = first.extend(second)
print(third)
assert third.x[:,0].sum() == 6
assert third.x[0,:].sum() == 6
assert third.x[:,2].sum() == 6
assert third.x[2,:].sum() == 6
assert third.x[:,3].sum() == 3
assert third.x[3,:].sum() == 3
assert third.x[:,6].sum() == 3
assert third.x[6,:].sum() == 3
def enforce_test():
import os
import pyemu
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"),
verbose=True)
cov = pyemu.Cov(x=mc.parcov.x * 0.1,
names=mc.parcov.row_names,
isdiagonal=True)
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"),
parcov=cov)
mc.draw(num_reals=100, enforce_bounds='drop')
assert mc.parensemble.shape[0] == 100.0
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"))
mc.draw(num_reals=100, enforce_bounds='drop')
assert mc.parensemble.shape[0] == 0
mc = pyemu.MonteCarlo(jco=os.path.join("mc", "freyberg_ord.jco"))
mc.draw(100, enforce_bounds="reset")
diff = mc.parensemble.ubnd - mc.parensemble.max(axis=0)
assert diff.min() == 0.0
diff = mc.parensemble.lbnd - mc.parensemble.min(axis=0)
assert diff.max() == 0.0
def sparse_extend_test():
import os
from datetime import datetime
import numpy as np
import pyemu
nrow = 5
ncol = 5
rnames = ["row_{0}".format(i) for i in range(nrow)]
cnames = ["col_{0}".format(i) for i in range(ncol)]
x = np.random.random((nrow, ncol))
m = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm = pyemu.SparseMatrix.from_matrix(m)
try:
sm.block_extend_ip(m)
except:
pass
else:
raise Exception()
m = pyemu.Matrix(x,
row_names=['t{0}'.format(i) for i in range(nrow)],
col_names=m.col_names)
try:
sm.block_extend_ip(m)
except:
pass
else:
raise Exception()
m = pyemu.Matrix(x,
row_names=['r{0}'.format(i) for i in range(nrow)],
col_names=['r{0}'.format(i) for i in range(ncol)])
sm.block_extend_ip(m)
m1 = sm.to_matrix()
d = m.x - m1.x[m.shape[0]:, m.shape[1]:]
assert d.sum() == 0
m = pyemu.Cov(x=np.atleast_2d(np.ones(nrow)),
names=['p{0}'.format(i) for i in range(nrow)],
isdiagonal=True)
sm.block_extend_ip(m)
d = m.as_2d - sm.to_matrix().x[-nrow:, -nrow:]
assert d.sum() == 0
m1 = pyemu.Matrix(x=x, row_names=rnames, col_names=cnames)
sm1 = pyemu.SparseMatrix.from_matrix(m1)
sm2 = pyemu.SparseMatrix.from_matrix(m)
sm1.block_extend_ip(sm2)
m2 = sm1.to_matrix()
d = m.as_2d - m2.x[m.shape[0]:, m.shape[1]:]
assert d.sum() == 0
def prep():
if os.path.exists(pyemu_dir):
shutil.rmtree(pyemu_dir)
if os.path.exists(pestpp_dir):
shutil.rmtree(pestpp_dir)
shutil.copytree("template", pyemu_dir)
shutil.copytree("template", pestpp_dir)
num_reals = 15
pst = pyemu.Pst(os.path.join(pyemu_dir, "pest.pst"))
pst.control_data.noptmax = 20
pst.pestpp_options["ies_parameter_csv"] = "par.csv"
pst.pestpp_options["ies_observation_csv"] = "obs.csv"
pst.pestpp_options["ies_obs_restart_csv"] = "restart_obs.csv"
pst.pestpp_options["ies_use_approx"] = "true"
pst.pestpp_options["ies_use_prior_scaling"] = "false"
pst.pestpp_options["ies_num_reals"] = "{0}".format(num_reals)
pst.pestpp_options["parcov_filename"] = "prior.cov"
pst.svd_data.eigthresh = 1.0e-5
pst.svd_data.maxsing = 20
#pst.observation_data.loc[pst.nnz_obs_names,"weight"] /= 10.0
pst.write(os.path.join(pyemu_dir, "pest.pst"))
pst.write(os.path.join(pestpp_dir, "pest.pst"))
#dia_parcov = pyemu.Cov.from_parameter_data(pst,sigma_range=6.0)
parcov = pyemu.Cov.from_parameter_data(pst)
print(parcov.as_2d)
parcov = pyemu.Cov(parcov.as_2d, names=parcov.row_names)
print(parcov.x)
parcov.to_ascii(os.path.join(pestpp_dir, "prior.cov"))
pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst,
parcov,
num_reals=num_reals,
use_homegrown=True)
oe = pyemu.ObservationEnsemble.from_id_gaussian_draw(pst,
num_reals=num_reals)
oe.to_csv(os.path.join(pyemu_dir, "obs.csv"))
oe.to_csv(os.path.join(pestpp_dir, "obs.csv"))
oe.to_binary(os.path.join(pestpp_dir, "obs.jcb"))
pe.to_csv(os.path.join(pyemu_dir, "sweep_in.csv"))
pe.to_csv(os.path.join(pyemu_dir, "par.csv"))
pe.to_csv(os.path.join(pestpp_dir, "par.csv"))
pe.to_binary(os.path.join(pestpp_dir, "par.jcb"))
pyemu.helpers.start_slaves(pyemu_dir,
"sweep",
"pest.pst",
num_slaves=5,
master_dir="sweep_master",
cleanup=False)
df = pd.read_csv(os.path.join("sweep_master", "sweep_out.csv"))
df.to_csv(os.path.join(pyemu_dir, "restart_obs.csv"))
df = df.loc[:, pst.observation_data.obsnme.apply(str.upper)]
df.to_csv(os.path.join(pestpp_dir, "restart_obs.csv"))
oe = pyemu.ObservationEnsemble.from_dataframe(df=df, pst=pst)
oe.to_binary(os.path.join(pestpp_dir, "restart_obs.jcb"))
def ensemble_change_test():
import matplotlib.pyplot as plt
pst = pyemu.Pst.from_par_obs_names(par_names=["p1","p2"])
cov = pyemu.Cov(np.array([[1.0,0.0],[0.0,1.0]]),names=["p1","p2"])
pe1 = pyemu.ParameterEnsemble.from_gaussian_draw(pst=pst,cov=cov,num_reals=5000)
cov *= 0.1
pe2 = pyemu.ParameterEnsemble.from_gaussian_draw(pst=pst,cov=cov,num_reals=5000)
pyemu.plot_utils.ensemble_change_summary(pe1,pe2,pst=pst)
print(pe1.mean(),pe1.std())
print(pe2.mean(),pe2.std())
pyemu.plot_utils.ensemble_change_summary(pe1,pe2,pst)
def cov_identity_test():
import os
import numpy as np
import pyemu
n = 100
names = ["name_{0}".format(i) for i in range(n)]
arr = np.random.random(n * n)
arr.resize((n, n))
cov = pyemu.Cov(x=arr * arr.transpose(), names=names)
cov *= 2.0
cov_i = pyemu.Cov.identity_like(cov)
cov_i *= 2.0
assert cov_i.x[0, 0] == 2.0
def enforce_test():
import os
import pyemu
dir = "mc"
mc = pyemu.MonteCarlo(jco=os.path.join("mc","freyberg_ord.jco"))
mc.draw(num_reals=100,enforce_bounds='drop')
assert mc.parensemble.shape[0] == 0
cov = pyemu.Cov(x=mc.parcov.x * 0.1,names=mc.parcov.row_names,isdiagonal=True)
mc = pyemu.MonteCarlo(jco=os.path.join("mc","freyberg_ord.jco"),
parcov=cov)
mc.draw(num_reals=100,enforce_bounds='drop')
assert mc.parensemble.shape[0] == 100.0
def get_test():
import numpy as np
import pyemu
arr = np.arange(0,12)
arr.resize(4,3)
first = pyemu.Jco(x=arr,col_names=["p2","p1","p3"],row_names=["o4","o1","o3","o2"])
#print(first)
#second = first.get(row_names=["o1","o3"])
second = first.get(row_names=first.row_names)
assert np.array_equal(first.x,second.x)
cov1 = pyemu.Cov(x=np.atleast_2d(np.arange(10)).transpose(),names=["c{0}".format(i) for i in range(10)],isdiagonal=True)
print(cov1)
cov2 = cov1.get(row_names=cov1.row_names)
print(cov2)
def basic_test(model_d="ies_10par_xsec"):
pyemu.Ensemble.reseed()
base_d = os.path.join(model_d, "template")
new_d = os.path.join(model_d, "test_template")
if os.path.exists(new_d):
shutil.rmtree(new_d)
shutil.copytree(base_d, new_d)
print(platform.platform().lower())
local=True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local=False
pst = pyemu.Pst(os.path.join(new_d, "pest.pst"))
print(pst.model_command)
# set first par as fixed
#pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed"
pst.observation_data.loc[pst.nnz_obs_names,"weight"] = 1.0
# set noptmax
pst.control_data.noptmax = noptmax
# wipe all pestpp options
pst.pestpp_options = {}
pst.pestpp_options["ies_num_reals"] = num_reals
pst.pestpp_options["lambda_scale_fac"] = [0.5,0.75,1.0]
pst.pestpp_options["ies_lambda_mults"] = 1.0
# write a generic 2D cov
if os.path.exists(os.path.join(new_d,"prior.jcb")):
cov = pyemu.Cov.from_binary(os.path.join(new_d,"prior.jcb"))
#cov.to_ascii(os.path.join(new_d,"prior.cov"))
elif os.path.exists(os.path.join(new_d, "prior.cov")):
cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov"))
else:
cov = pyemu.Cov.from_parameter_data(pst)
cov = pyemu.Cov(cov.as_2d, names=cov.row_names)
#cov.to_ascii(os.path.join(new_d, "prior.cov"))
cov.to_binary(os.path.join(new_d, "prior.jcb"))
# draw some ensembles
idx = [i for i in range(num_reals)]
idx[-1] = "base"
pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov=cov, num_reals=num_reals)
pe.index = idx
pe.to_csv(os.path.join(new_d, "par.csv"))
pe.to_binary(os.path.join(new_d, "par.jcb"))
pe.to_csv(os.path.join(new_d, "sweep_in.csv"))
pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv"))
pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv"))
oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals)
oe.index = idx
oe.to_csv(os.path.join(new_d, "obs.csv"))
oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv"))
oe.to_binary(os.path.join(new_d, "obs.jcb"))
pst.control_data.noptmax = 0
pst.write(os.path.join(new_d, "pest.pst"))
pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d)
df = pd.read_csv(os.path.join(new_d,"pest.phi.group.csv"))
assert df.loc[0,"head"] == 0.5,df
#return
pst.control_data.noptmax = noptmax
pst.write(os.path.join(new_d, "pest.pst"))
m_d = os.path.join(model_d,"master_pestpp_sen")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-sen"), "pest.pst", 5, master_dir=m_d,
worker_root=model_d,local=local,port=port)
df = pd.read_csv(os.path.join(m_d, "pest.mio"),index_col=0)
# run sweep
m_d = os.path.join(model_d,"master_sweep1")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-swp"), "pest.pst", 5, master_dir=m_d,
worker_root=model_d,local=local,port=port)
df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0)
m_d = os.path.join(model_d,"master_pestpp-glm")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-glm"), "pest.pst", 10, master_dir=m_d,
worker_root=model_d,local=local,port=port)
m_d = os.path.join(model_d,"master_pestpp-ies")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pyemu.os_utils.start_workers(new_d, exe_path, "pest.pst", 10, master_dir=m_d,
worker_root=model_d,local=local,port=port)
def setup_suite_dir(model_d):
pyemu.Ensemble.reseed()
base_d = os.path.join(model_d, "template")
new_d = os.path.join(model_d, "test_template")
if os.path.exists(new_d):
shutil.rmtree(new_d)
shutil.copytree(base_d, new_d)
print(platform.platform().lower())
local = True
if "linux" in platform.platform().lower() and "10par" in model_d:
#print("travis_prep")
#prep_for_travis(model_d)
local = False
pst = pyemu.Pst(os.path.join(new_d, "pest.pst"))
print(pst.model_command)
# set first par as fixed
#pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed"
pst.observation_data.loc[pst.nnz_obs_names, "weight"] = 1.0
# set noptmax
pst.control_data.noptmax = noptmax
# wipe all pestpp options
pst.pestpp_options = {}
# write a generic 2D cov
if os.path.exists(os.path.join(new_d, "prior.jcb")):
cov = pyemu.Cov.from_binary(os.path.join(new_d, "prior.jcb"))
#cov.to_ascii(os.path.join(new_d,"prior.cov"))
elif os.path.exists(os.path.join(new_d, "prior.cov")):
cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov"))
else:
cov = pyemu.Cov.from_parameter_data(pst)
cov = pyemu.Cov(cov.as_2d, names=cov.row_names)
#cov.to_ascii(os.path.join(new_d, "prior.cov"))
cov.to_binary(os.path.join(new_d, "prior.jcb"))
# draw some ensembles
idx = [i for i in range(num_reals)]
idx[-1] = "base"
pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst,
cov=cov,
num_reals=num_reals)
pe.index = idx
pe.to_csv(os.path.join(new_d, "par.csv"))
pe.to_binary(os.path.join(new_d, "par.jcb"))
pe.to_csv(os.path.join(new_d, "sweep_in.csv"))
pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv"))
pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv"))
oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals)
oe.index = idx
oe.to_csv(os.path.join(new_d, "obs.csv"))
oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv"))
oe.to_binary(os.path.join(new_d, "obs.jcb"))
pst.write(os.path.join(new_d, "pest.pst"))
# run sweep
m_d = os.path.join(model_d, "master_sweep1")
if os.path.exists(m_d):
shutil.rmtree(m_d)
pyemu.os_utils.start_workers(new_d,
exe_path.replace("-ies", "-swp"),
"pest.pst",
5,
master_dir=m_d,
worker_root=model_d,
local=local,
port=port)
#shutil.copytree(new_d,m_d)
#pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-swp")),cwd=m_d)
# process sweep output as restart csv and jcb
df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"), index_col=0)
df.index = df.input_run_id
df.columns = [c.lower() for c in df.columns]
df.to_csv(os.path.join(new_d, "restart.csv"))
df = df.loc[:, pst.nnz_obs_names]
df.loc[:, pst.nnz_obs_names].to_csv(os.path.join(new_d,
"restart_some.csv"))
df.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed.csv"))