def pe_to_csv_test(): import os import numpy as np import pandas as pd import pyemu from pyemu import MonteCarlo jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") pst = pyemu.Pst(pst) #pst.parameter_data = pst.parameter_data.iloc[:2,:] pst.parameter_data.loc["mult1","partrans"] = "none" pst.parameter_data.loc["mult1","parval1"] = -1.0 mc = MonteCarlo(pst=pst,verbose=True) mc.draw(1,enforce_bounds="reset") if not mc.parensemble.istransformed: mc.parensemble._transform() fname = os.path.join("temp","test.csv") mc.parensemble.to_csv(fname) df = pd.read_csv(fname) pe = pyemu.ParameterEnsemble.from_dataframe(pst=pst,df=df) pe1 = pe.copy() pe.enforce() assert np.allclose(pe1.as_matrix(),pe.as_matrix())
def add_base_test(): import os import numpy as np from pyemu import MonteCarlo, Cov, ParameterEnsemble from datetime import datetime jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) num_reals = 100 mc.draw(num_reals=num_reals, how="gaussian",obs=True) mc.parensemble.add_base() diff = mc.parensemble.loc["base",:] - mc.pst.parameter_data.parval1 assert diff.sum() == 0.0 try: mc.parensemble.add_base() except: pass else: raise Exception() mc.obsensemble.add_base() diff = mc.obsensemble.loc["base", :] - mc.pst.observation_data.obsval assert diff.sum() == 0.0 try: mc.obsensemble.add_base() except: pass else: raise Exception()
def parfile_test(): import os import numpy as np import pandas as pd from pyemu import MonteCarlo, Ensemble, ParameterEnsemble, Pst, Cov jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) mc.pst.parameter_data.loc[mc.pst.par_names[1], "scale"] = 0.001 mc.draw(10) mc.parensemble.to_parfiles(os.path.join("temp", "testpar")) pst = Pst(pst) pst.parameter_data = pst.parameter_data.iloc[1:] pst.parameter_data["test", "parmne"] = "test" parfiles = [ os.path.join("temp", f) for f in os.listdir("temp") if "testpar" in f ] rnames = ["test{0}".format(i) for i in range(len(parfiles))] pe = ParameterEnsemble.from_parfiles(pst=pst, parfile_names=parfiles, real_names=rnames)
def gaussian_draw_test(): import os import numpy as np from pyemu import MonteCarlo,Cov,ParameterEnsemble from datetime import datetime jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) num_reals = 100 start = datetime.now() mc.draw(num_reals=num_reals,how="gaussian") print(mc.parensemble.head()) print(datetime.now() - start) vals = mc.pst.parameter_data.parval1.values cov = Cov.from_parameter_data(mc.pst) start = datetime.now() val_array = np.random.multivariate_normal(vals, cov.as_2d,num_reals) print(datetime.now() - start) start = datetime.now() pe = ParameterEnsemble.from_gaussian_draw(mc.pst,cov,num_reals=num_reals) pet = pe._transform(inplace=False) pe = pet._back_transform(inplace=False) print(datetime.now() - start) print(mc.parensemble.head()) print(pe.head())
def fixed_par_test(): import os import numpy as np from pyemu import MonteCarlo jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) mc.pst.parameter_data.loc["mult1","partrans"] = "fixed" mc.draw(10) assert np.all(mc.parensemble.loc[:,"mult1"] == mc.pst.parameter_data.loc["mult1","parval1"])
def enforce_scale(): import os import pyemu from pyemu import MonteCarlo jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") pst = pyemu.Pst(pst) pst.parameter_data = pst.parameter_data.iloc[:2,:] pst.parameter_data.loc["mult1","partrans"] = "none" pst.parameter_data.loc["mult1","parval1"] = -1.0 mc = MonteCarlo(pst=pst,verbose=True) mc.draw(1,enforce_bounds="scale")
def from_dataframe_test(): import os import numpy as np import pandas as pd from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) names = ["par_{0}".format(_) for _ in range(10)] df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names) mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst) print(mc.parensemble.shape) mc.project_parensemble() mc.parensemble.to_csv(os.path.join("temp","test.csv"))
def uniform_draw_test(): import os import numpy as np from pyemu import MonteCarlo jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) from datetime import datetime start = datetime.now() mc.draw(num_reals=1000,how="uniform") print(datetime.now() - start) import matplotlib.pyplot as plt ax = mc.parensemble.loc[:,"mult1"].plot(kind="hist",bins=50,alpha=0.5) mc.draw(num_reals=1000) mc.parensemble.loc[:,"mult1"].plot(kind="hist",bins=50,ax=ax,alpha=0.5)
def change_weights_test(): import os import numpy as np import pyemu from pyemu import MonteCarlo, ObservationEnsemble from datetime import datetime jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) print(mc.pst.nnz_obs_names) ogcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names, mc.pst.nnz_obs_names] num_reals = 10000 oe = ObservationEnsemble.from_id_gaussian_draw(mc.obsensemble, num_reals=num_reals) for oname in mc.pst.nnz_obs_names: w = mc.pst.observation_data.loc[oname, "weight"] v = ogcov.loc[oname, oname] est = np.std(oe.loc[:, oname])**2 pd = 100.0 * (np.abs(v - est)) / v print(oname, np.std(oe.loc[:, oname])**2, ogcov.loc[oname, oname], pd, (1.0 / w)**2) assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd) assert (1.0 / w)**2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w)**2) mc.pst.observation_data.loc[mc.pst.nnz_obs_names, "weight"] = 1000.0 mc.reset_obscov(pyemu.Cov.from_observation_data(mc.pst)) newcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names, mc.pst.nnz_obs_names] #print(mc.obsensemble.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"]) num_reals = 10000 oe = ObservationEnsemble.from_id_gaussian_draw(mc.obsensemble, num_reals=num_reals) for oname in mc.pst.nnz_obs_names: w = mc.pst.observation_data.loc[oname, "weight"] v = newcov.loc[oname, oname] est = np.std(oe.loc[:, oname])**2 pd = 100.0 * (np.abs(v - est)) / v # print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2) assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd) assert (1.0 / w)**2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w)**2)
def from_dataframe_test(): import os import numpy as np import pandas as pd from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst, Cov jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) names = ["par_{0}".format(_) for _ in range(10)] df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names) mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst) print(mc.parensemble.shape) mc.project_parensemble() mc.parensemble.to_csv(os.path.join("temp","test.csv")) pstc = Pst(pst) par = pstc.parameter_data par.sort_values(by="parnme",ascending=False,inplace=True) cov = Cov.from_parameter_data(pstc) pe = ParameterEnsemble.from_gaussian_draw(pst=mc.pst,cov=cov)
def change_weights_test(): import os import numpy as np import pyemu from pyemu import MonteCarlo, ObservationEnsemble from datetime import datetime jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) print(mc.pst.nnz_obs_names) ogcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,mc.pst.nnz_obs_names] num_reals = 10000 oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst, num_reals=num_reals) for oname in mc.pst.nnz_obs_names: w = mc.pst.observation_data.loc[oname,"weight"] v = ogcov.loc[oname,oname] est = np.std(oe.loc[:,oname])**2 pd = 100.0 * (np.abs(v-est)) / v print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2) assert pd < 10.0,"{0},{1},{2},{3}".format(oname,v,est,pd) assert (1.0/w)**2 == v,"{0},{1},{2}".format(oname,v,(1.0/w)**2) mc.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"] = 1000.0 mc.reset_obscov(pyemu.Cov.from_observation_data(mc.pst)) newcov = mc.obscov.to_dataframe().loc[mc.pst.nnz_obs_names,mc.pst.nnz_obs_names] #print(mc.obsensemble.pst.observation_data.loc[mc.pst.nnz_obs_names,"weight"]) num_reals = 10000 oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst, num_reals=num_reals) for oname in mc.pst.nnz_obs_names: w = mc.pst.observation_data.loc[oname, "weight"] v = newcov.loc[oname, oname] est = np.std(oe.loc[:, oname]) ** 2 pd = 100.0 * (np.abs(v - est)) / v # print(oname,np.std(oe.loc[:,oname])**2,ogcov.loc[oname,oname],pd,(1.0/w)**2) assert pd < 10.0, "{0},{1},{2},{3}".format(oname, v, est, pd) assert (1.0 / w) ** 2 == v, "{0},{1},{2}".format(oname, v, (1.0 / w) ** 2)
def obs_id_draw_test(): import os import numpy as np from pyemu import MonteCarlo,ObservationEnsemble from datetime import datetime jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) num_reals = 100 oe = ObservationEnsemble.from_id_gaussian_draw(mc.pst,num_reals=num_reals) print(oe.shape) print(oe.head())
def parfile_test(): import os import numpy as np import pandas as pd from pyemu import MonteCarlo, Ensemble, ParameterEnsemble, Pst, Cov jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) mc.pst.parameter_data.loc[mc.pst.par_names[1], "scale"] = 0.001 mc.draw(10) mc.parensemble.to_parfiles(os.path.join("temp","testpar")) pst = Pst(pst) pst.parameter_data = pst.parameter_data.iloc[1:] pst.parameter_data["test","parmne"] = "test" parfiles = [os.path.join("temp",f) for f in os.listdir("temp") if "testpar" in f] rnames = ["test{0}".format(i) for i in range(len(parfiles))] pe = ParameterEnsemble.from_parfiles(pst=pst,parfile_names=parfiles,real_names=rnames)
def write_regul_test(): import os import numpy as np from pyemu import MonteCarlo mc = MonteCarlo(jco=os.path.join("verf_results","freyberg_ord.jco")) mc.pst.control_data.pestmode = "regularization" mc.draw(10) mc.write_psts(os.path.join("temp","freyberg_real"),existing_jco="freyberg_ord.jco")
def diagonal_cov_draw_test(): import os import numpy as np from pyemu import MonteCarlo, Cov, Pst jco = os.path.join("pst", "pest.jcb") pst = Pst(jco.replace(".jcb", ".pst")) mc = MonteCarlo(jco=jco, pst=pst) num_reals = 10 mc.draw(num_reals, obs=True) print(mc.obsensemble) pe1 = mc.parensemble.copy() cov = Cov(x=mc.parcov.as_2d, names=mc.parcov.row_names) #print(type(cov)) mc = MonteCarlo(jco=jco, pst=pst) mc.parensemble.reseed() mc.draw(num_reals, cov=cov) pe2 = mc.parensemble
def uniform_draw_test(): import os import numpy as np from pyemu import MonteCarlo jco = os.path.join("pst", "pest.jcb") pst = jco.replace(".jcb", ".pst") mc = MonteCarlo(jco=jco, pst=pst) from datetime import datetime start = datetime.now() mc.draw(num_reals=1000, how="uniform") print(datetime.now() - start) import matplotlib.pyplot as plt ax = mc.parensemble.loc[:, "mult1"].plot(kind="hist", bins=50, alpha=0.5) mc.draw(num_reals=1000) mc.parensemble.loc[:, "mult1"].plot(kind="hist", bins=50, ax=ax, alpha=0.5)
def diagonal_cov_draw_test(): import os import numpy as np from pyemu import MonteCarlo,Cov,Pst,ParameterEnsemble jco = os.path.join("pst","pest.jcb") pst = Pst(jco.replace(".jcb",".pst")) mc = MonteCarlo(jco=jco,pst=pst) num_reals = 100 mc.draw(num_reals,obs=True) print(mc.obsensemble) pe1 = mc.parensemble.copy() cov = Cov(x=mc.parcov.as_2d,names=mc.parcov.row_names) #print(type(cov)) mc = MonteCarlo(jco=jco,pst=pst) mc.parensemble.reseed() mc.draw(num_reals,cov=cov) pe2 = mc.parensemble pe3 = ParameterEnsemble.from_gaussian_draw(mc.pst,num_reals=num_reals,cov=mc.parcov)
def mc_test(): import os import numpy as np from pyemu import MonteCarlo, Cov jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") out_dir = os.path.join("mc") if not os.path.exists(out_dir): os.mkdir(out_dir) #write testing mc = MonteCarlo(jco=jco,verbose=True,sigma_range=6) cov = Cov.from_parameter_data(mc.pst,sigma_range=6) assert np.abs((mc.parcov.x - cov.x).sum()) == 0.0 mc.draw(10,obs=True) mc.write_psts(os.path.join("temp","real_")) mc.parensemble.to_parfiles(os.path.join("mc","real_")) mc = MonteCarlo(jco=jco,verbose=True) mc.draw(10,obs=True) print("prior ensemble variance:", np.var(mc.parensemble.loc[:,"mult1"])) projected_en = mc.project_parensemble(inplace=False) print("projected ensemble variance:", np.var(projected_en.loc[:,"mult1"])) import pyemu sc = pyemu.Schur(jco=jco) mc = MonteCarlo(pst=pst,parcov=sc.posterior_parameter,verbose=True) mc.draw(10) print("posterior ensemble variance:", np.var(mc.parensemble.loc[:,"mult1"]))
from pyemu import MonteCarlo mc = MonteCarlo(pst="pest.pst") mc.draw(50) mc.parensemble.to_csv("sweep_in.csv")