def pe_to_csv_test():
import os
import numpy as np
import pandas as pd
import pyemu
from pyemu import MonteCarlo
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
pst = pyemu.Pst(pst)
#pst.parameter_data = pst.parameter_data.iloc[:2,:]
pst.parameter_data.loc["mult1","partrans"] = "none"
pst.parameter_data.loc["mult1","parval1"] = -1.0
mc = MonteCarlo(pst=pst,verbose=True)
mc.draw(1,enforce_bounds="reset")
if not mc.parensemble.istransformed:
mc.parensemble._transform()
fname = os.path.join("temp","test.csv")
mc.parensemble.to_csv(fname)
df = pd.read_csv(fname)
pe = pyemu.ParameterEnsemble.from_dataframe(pst=pst,df=df)
pe1 = pe.copy()
pe.enforce()
assert np.allclose(pe1.as_matrix(),pe.as_matrix())
def add_base_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov, ParameterEnsemble
from datetime import datetime
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
num_reals = 100
mc.draw(num_reals=num_reals, how="gaussian",obs=True)
mc.parensemble.add_base()
diff = mc.parensemble.loc["base",:] - mc.pst.parameter_data.parval1
assert diff.sum() == 0.0
try:
mc.parensemble.add_base()
except:
pass
else:
raise Exception()
mc.obsensemble.add_base()
diff = mc.obsensemble.loc["base", :] - mc.pst.observation_data.obsval
assert diff.sum() == 0.0
try:
mc.obsensemble.add_base()
except:
pass
else:
raise Exception()
def parfile_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo, Ensemble, ParameterEnsemble, Pst, Cov
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
mc.pst.parameter_data.loc[mc.pst.par_names[1], "scale"] = 0.001
mc.draw(10)
mc.parensemble.to_parfiles(os.path.join("temp", "testpar"))
pst = Pst(pst)
pst.parameter_data = pst.parameter_data.iloc[1:]
pst.parameter_data["test", "parmne"] = "test"
parfiles = [
os.path.join("temp", f) for f in os.listdir("temp") if "testpar" in f
]
rnames = ["test{0}".format(i) for i in range(len(parfiles))]
pe = ParameterEnsemble.from_parfiles(pst=pst,
parfile_names=parfiles,
real_names=rnames)
def gaussian_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo,Cov,ParameterEnsemble
from datetime import datetime
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
num_reals = 100
start = datetime.now()
mc.draw(num_reals=num_reals,how="gaussian")
print(mc.parensemble.head())
print(datetime.now() - start)
vals = mc.pst.parameter_data.parval1.values
cov = Cov.from_parameter_data(mc.pst)
start = datetime.now()
val_array = np.random.multivariate_normal(vals, cov.as_2d,num_reals)
print(datetime.now() - start)
start = datetime.now()
pe = ParameterEnsemble.from_gaussian_draw(mc.pst,cov,num_reals=num_reals)
pet = pe._transform(inplace=False)
pe = pet._back_transform(inplace=False)
print(datetime.now() - start)
print(mc.parensemble.head())
print(pe.head())
def fixed_par_test():
import os
import numpy as np
from pyemu import MonteCarlo
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
mc.pst.parameter_data.loc["mult1","partrans"] = "fixed"
mc.draw(10)
assert np.all(mc.parensemble.loc[:,"mult1"] ==
mc.pst.parameter_data.loc["mult1","parval1"])
def fixed_par_test():
import os
import numpy as np
from pyemu import MonteCarlo
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
mc.pst.parameter_data.loc["mult1","partrans"] = "fixed"
mc.draw(10)
assert np.all(mc.parensemble.loc[:,"mult1"] ==
mc.pst.parameter_data.loc["mult1","parval1"])
def enforce_scale():
import os
import pyemu
from pyemu import MonteCarlo
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
pst = pyemu.Pst(pst)
pst.parameter_data = pst.parameter_data.iloc[:2,:]
pst.parameter_data.loc["mult1","partrans"] = "none"
pst.parameter_data.loc["mult1","parval1"] = -1.0
mc = MonteCarlo(pst=pst,verbose=True)
mc.draw(1,enforce_bounds="scale")
def from_dataframe_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
names = ["par_{0}".format(_) for _ in range(10)]
df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names)
mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst)
print(mc.parensemble.shape)
mc.project_parensemble()
mc.parensemble.to_csv(os.path.join("temp","test.csv"))
def uniform_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
from datetime import datetime
start = datetime.now()
mc.draw(num_reals=1000,how="uniform")
print(datetime.now() - start)
import matplotlib.pyplot as plt
ax = mc.parensemble.loc[:,"mult1"].plot(kind="hist",bins=50,alpha=0.5)
mc.draw(num_reals=1000)
mc.parensemble.loc[:,"mult1"].plot(kind="hist",bins=50,ax=ax,alpha=0.5)
def from_dataframe_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
names = ["par_{0}".format(_) for _ in range(10)]
df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names)
mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst)
print(mc.parensemble.shape)
mc.project_parensemble()
mc.parensemble.to_csv(os.path.join("temp","test.csv"))
def change_weights_test():
import os
import numpy as np
import pyemu
from pyemu import MonteCarlo, ObservationEnsemble
from datetime import datetime
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
print(mc.pst.nnz_obs_names)
ogcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,
mc.pst.nnz_obs_names]
num_reals = 10000
oe = ObservationEnsemble.from_id_gaussian_draw(mc.obsensemble,
num_reals=num_reals)
for oname in mc.pst.nnz_obs_names:
w = mc.pst.observation_data.loc[oname, "weight"]
v = ogcov.loc[oname, oname]
est = np.std(oe.loc[:, oname])**2
pd = 100.0 * (np.abs(v - est)) / v
print(oname,
np.std(oe.loc[:, oname])**2, ogcov.loc[oname, oname], pd,
(1.0 / w)**2)
assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd)
assert (1.0 / w)**2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w)**2)
mc.pst.observation_data.loc[mc.pst.nnz_obs_names, "weight"] = 1000.0
mc.reset_obscov(pyemu.Cov.from_observation_data(mc.pst))
newcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,
mc.pst.nnz_obs_names]
#print(mc.obsensemble.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"])
num_reals = 10000
oe = ObservationEnsemble.from_id_gaussian_draw(mc.obsensemble,
num_reals=num_reals)
for oname in mc.pst.nnz_obs_names:
w = mc.pst.observation_data.loc[oname, "weight"]
v = newcov.loc[oname, oname]
est = np.std(oe.loc[:, oname])**2
pd = 100.0 * (np.abs(v - est)) / v
# print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2)
assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd)
assert (1.0 / w)**2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w)**2)
def from_dataframe_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst, Cov
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
names = ["par_{0}".format(_) for _ in range(10)]
df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names)
mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst)
print(mc.parensemble.shape)
mc.project_parensemble()
mc.parensemble.to_csv(os.path.join("temp","test.csv"))
pstc = Pst(pst)
par = pstc.parameter_data
par.sort_values(by="parnme",ascending=False,inplace=True)
cov = Cov.from_parameter_data(pstc)
pe = ParameterEnsemble.from_gaussian_draw(pst=mc.pst,cov=cov)
def change_weights_test():
import os
import numpy as np
import pyemu
from pyemu import MonteCarlo, ObservationEnsemble
from datetime import datetime
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
print(mc.pst.nnz_obs_names)
ogcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,mc.pst.nnz_obs_names]
num_reals = 10000
oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst, num_reals=num_reals)
for oname in mc.pst.nnz_obs_names:
w = mc.pst.observation_data.loc[oname,"weight"]
v = ogcov.loc[oname,oname]
est = np.std(oe.loc[:,oname])**2
pd = 100.0 * (np.abs(v-est)) / v
print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2)
assert pd < 10.0,"{0},{1},{2},{3}".format(oname,v,est,pd)
assert (1.0/w)**2 == v,"{0},{1},{2}".format(oname,v,(1.0/w)**2)
mc.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"] = 1000.0
mc.reset_obscov(pyemu.Cov.from_observation_data(mc.pst))
newcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,mc.pst.nnz_obs_names]
#print(mc.obsensemble.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"])
num_reals = 10000
oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst, num_reals=num_reals)
for oname in mc.pst.nnz_obs_names:
w = mc.pst.observation_data.loc[oname, "weight"]
v = newcov.loc[oname, oname]
est = np.std(oe.loc[:, oname]) ** 2
pd = 100.0 * (np.abs(v - est)) / v
# print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2)
assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd)
assert (1.0 / w) ** 2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w) ** 2)
def obs_id_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo,ObservationEnsemble
from datetime import datetime
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
num_reals = 100
oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst,num_reals=num_reals)
print(oe.shape)
print(oe.head())
def parfile_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo, Ensemble, ParameterEnsemble, Pst, Cov
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
mc.pst.parameter_data.loc[mc.pst.par_names[1], "scale"] = 0.001
mc.draw(10)
mc.parensemble.to_parfiles(os.path.join("temp","testpar"))
pst = Pst(pst)
pst.parameter_data = pst.parameter_data.iloc[1:]
pst.parameter_data["test","parmne"] = "test"
parfiles = [os.path.join("temp",f) for f in os.listdir("temp") if "testpar" in f]
rnames = ["test{0}".format(i) for i in range(len(parfiles))]
pe = ParameterEnsemble.from_parfiles(pst=pst,parfile_names=parfiles,real_names=rnames)
def write_regul_test():
import os
import numpy as np
from pyemu import MonteCarlo
mc = MonteCarlo(jco=os.path.join("verf_results","freyberg_ord.jco"))
mc.pst.control_data.pestmode = "regularization"
mc.draw(10)
mc.write_psts(os.path.join("temp","freyberg_real"),existing_jco="freyberg_ord.jco")
def diagonal_cov_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov, Pst
jco = os.path.join("pst", "pest.jcb")
pst = Pst(jco.replace(".jcb", ".pst"))
mc = MonteCarlo(jco=jco, pst=pst)
num_reals = 10
mc.draw(num_reals, obs=True)
print(mc.obsensemble)
pe1 = mc.parensemble.copy()
cov = Cov(x=mc.parcov.as_2d, names=mc.parcov.row_names)
#print(type(cov))
mc = MonteCarlo(jco=jco, pst=pst)
mc.parensemble.reseed()
mc.draw(num_reals, cov=cov)
pe2 = mc.parensemble
def uniform_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo
jco = os.path.join("pst", "pest.jcb")
pst = jco.replace(".jcb", ".pst")
mc = MonteCarlo(jco=jco, pst=pst)
from datetime import datetime
start = datetime.now()
mc.draw(num_reals=1000, how="uniform")
print(datetime.now() - start)
import matplotlib.pyplot as plt
ax = mc.parensemble.loc[:, "mult1"].plot(kind="hist", bins=50, alpha=0.5)
mc.draw(num_reals=1000)
mc.parensemble.loc[:, "mult1"].plot(kind="hist", bins=50, ax=ax, alpha=0.5)
def diagonal_cov_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo,Cov,Pst,ParameterEnsemble
jco = os.path.join("pst","pest.jcb")
pst = Pst(jco.replace(".jcb",".pst"))
mc = MonteCarlo(jco=jco,pst=pst)
num_reals = 100
mc.draw(num_reals,obs=True)
print(mc.obsensemble)
pe1 = mc.parensemble.copy()
cov = Cov(x=mc.parcov.as_2d,names=mc.parcov.row_names)
#print(type(cov))
mc = MonteCarlo(jco=jco,pst=pst)
mc.parensemble.reseed()
mc.draw(num_reals,cov=cov)
pe2 = mc.parensemble
pe3 = ParameterEnsemble.from_gaussian_draw(mc.pst,num_reals=num_reals,cov=mc.parcov)
def mc_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
out_dir = os.path.join("mc")
if not os.path.exists(out_dir):
os.mkdir(out_dir)
#write testing
mc = MonteCarlo(jco=jco,verbose=True,sigma_range=6)
cov = Cov.from_parameter_data(mc.pst,sigma_range=6)
assert np.abs((mc.parcov.x - cov.x).sum()) == 0.0
mc.draw(10,obs=True)
mc.write_psts(os.path.join("temp","real_"))
mc.parensemble.to_parfiles(os.path.join("mc","real_"))
mc = MonteCarlo(jco=jco,verbose=True)
mc.draw(10,obs=True)
print("prior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
projected_en = mc.project_parensemble(inplace=False)
print("projected ensemble variance:",
np.var(projected_en.loc[:,"mult1"]))
import pyemu
sc = pyemu.Schur(jco=jco)
mc = MonteCarlo(pst=pst,parcov=sc.posterior_parameter,verbose=True)
mc.draw(10)
print("posterior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
from pyemu import MonteCarlo
mc = MonteCarlo(pst="pest.pst")
mc.draw(50)
mc.parensemble.to_csv("sweep_in.csv")
