def covariance_matrix(self, x, y, names=None, cov=None):
"""build a pyemu.Cov instance from Vario2d
Parameters
----------
x : iterable of floats
x locations
y : iterable of floats
y locations
names : iterable of str (optional)
names of location. If None, generic names will be used
cov : pyemu.Cov instance (optional)
an existing Cov instance to add contribution to
Returns
-------
pyemu.Cov
"""
if not isinstance(x, np.ndarray):
x = np.array(x)
if not isinstance(y, np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names), len(names)))
np.fill_diagonal(c, self.contribution)
cov = Cov(x=c, names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names), 1)) + self.contribution
cont = Cov(x=c, names=names, isdiagonal=True)
cov += cont
else:
raise Exception("Vario2d.covariance_matrix() requires either" +
"names or cov arg")
for i1, (n1, x1, y1) in enumerate(zip(names, x, y)):
dx = x1 - x[i1 + 1:]
dy = y1 - y[i1 + 1:]
dxx = (dx * self.rotation_coefs[0]) +\
(dy * self.rotation_coefs[1])
dyy = ((dx * self.rotation_coefs[2]) +\
(dy * self.rotation_coefs[3])) *\
self.anisotropy
h = np.sqrt(dxx * dxx + dyy * dyy)
h[h < 0.0] = 0.0
h = self.h_function(h)
if np.any(np.isnan(h)):
raise Exception("nans in h for i1 {0}".format(i1))
cov.x[i1, i1 + 1:] += h
for i in range(len(names)):
cov.x[i + 1:, i] = cov.x[i, i + 1:]
return cov
def schur_test_nonpest():
import numpy as np
from pyemu import Matrix, Cov, Schur, Jco
#non-pest
pnames = ["p1", "p2", "p3"]
onames = ["o1", "o2", "o3", "o4"]
npar = len(pnames)
nobs = len(onames)
j_arr = np.random.random((nobs, npar))
jco = Jco(x=j_arr, row_names=onames, col_names=pnames)
parcov = Cov(x=np.eye(npar), names=pnames)
obscov = Cov(x=np.eye(nobs), names=onames)
forecasts = "o2"
s = Schur(jco=jco, parcov=parcov, obscov=obscov, forecasts=forecasts)
print(s.get_parameter_summary())
print(s.get_forecast_summary())
#this should fail
passed = False
try:
print(s.get_par_group_contribution())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
#this should fail
passed = False
try:
print(s.get_removed_obs_group_importance())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
print(s.get_par_contribution({"group1": ["p1", "p3"]}))
print(s.get_removed_obs_importance({"group1": ["o1", "o3"]}))
print(s.pandas)
forecasts = Matrix(x=np.random.random((1, npar)),
row_names=[forecasts],
col_names=pnames)
sc = Schur(jco=jco, forecasts=forecasts.T, parcov=parcov, obscov=obscov)
ffile = os.path.join("temp", "forecasts.jcb")
forecasts.to_binary(ffile)
sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov)
def covariance_matrix(self,x,y,names=None,cov=None):
"""build a pyemu.Cov instance from Vario2d
Parameters
----------
x : iterable of floats
x locations
y : iterable of floats
y locations
names : iterable of str (optional)
names of location. If None, generic names will be used
cov : pyemu.Cov instance (optional)
an existing Cov instance to add contribution to
Returns
-------
pyemu.Cov
"""
if not isinstance(x,np.ndarray):
x = np.array(x)
if not isinstance(y,np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names),len(names)))
np.fill_diagonal(c,self.contribution)
cov = Cov(x=c,names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names),1)) + self.contribution
cont = Cov(x=c,names=names,isdiagonal=True)
cov += cont
else:
raise Exception("Vario2d.covariance_matrix() requires either" +
"names or cov arg")
for i1,(n1,x1,y1) in enumerate(zip(names,x,y)):
dx = x1 - x[i1+1:]
dy = y1 - y[i1+1:]
dxx = (dx * self.rotation_coefs[0]) +\
(dy * self.rotation_coefs[1])
dyy = ((dx * self.rotation_coefs[2]) +\
(dy * self.rotation_coefs[3])) *\
self.anisotropy
h = np.sqrt(dxx*dxx + dyy*dyy)
h[h<0.0] = 0.0
cov.x[i1,i1+1:] += self.h_function(h)
for i in range(len(names)):
cov.x[i+1:,i] = cov.x[i,i+1:]
return cov
def schur_test():
import os
import numpy as np
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc"))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
df = sc.get_par_group_contribution(include_prior_results=True)
levels = list(df.columns.levels[1])
assert "prior" in levels, levels
assert "post" in levels, levels
print(sc.get_parameter_summary(include_map=True))
print(sc.get_forecast_summary(include_map=True))
print(sc.get_removed_obs_importance(reset_zero_weight=True))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0)
cov = Cov.from_parameter_data(pst, sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0,
scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
pst.parameter_data.loc[:, "offset"] = 100.0
cov = Cov.from_parameter_data(pst)
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
pst=pst,
forecasts=forecasts,
sigma_range=6.0,
scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0
cov = Cov.from_parameter_data(pst, scale_offset=False, sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def gaussian_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo,Cov,ParameterEnsemble
from datetime import datetime
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
num_reals = 100
start = datetime.now()
mc.draw(num_reals=num_reals,how="gaussian")
print(mc.parensemble.head())
print(datetime.now() - start)
vals = mc.pst.parameter_data.parval1.values
cov = Cov.from_parameter_data(mc.pst)
start = datetime.now()
val_array = np.random.multivariate_normal(vals, cov.as_2d,num_reals)
print(datetime.now() - start)
start = datetime.now()
pe = ParameterEnsemble.from_gaussian_draw(mc.pst,cov,num_reals=num_reals)
pet = pe._transform(inplace=False)
pe = pet._back_transform(inplace=False)
print(datetime.now() - start)
print(mc.parensemble.head())
print(pe.head())
def schur_test():
import os
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc"))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_removed_obs_group_importance())
def mc_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
out_dir = os.path.join("mc")
if not os.path.exists(out_dir):
os.mkdir(out_dir)
#write testing
mc = MonteCarlo(jco=jco,verbose=True,sigma_range=6)
cov = Cov.from_parameter_data(mc.pst,sigma_range=6)
assert np.abs((mc.parcov.x - cov.x).sum()) == 0.0
mc.draw(10,obs=True)
mc.write_psts(os.path.join("temp","real_"))
mc.parensemble.to_parfiles(os.path.join("mc","real_"))
mc = MonteCarlo(jco=jco,verbose=True)
mc.draw(10,obs=True)
print("prior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
projected_en = mc.project_parensemble(inplace=False)
print("projected ensemble variance:",
np.var(projected_en.loc[:,"mult1"]))
import pyemu
sc = pyemu.Schur(jco=jco)
mc = MonteCarlo(pst=pst,parcov=sc.posterior_parameter,verbose=True)
mc.draw(10)
print("posterior ensemble variance:",
np.var(mc.parensemble.loc[:,"mult1"]))
def schur_test():
import os
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..","..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
pst = Pst(os.path.join(w_dir,"pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc"))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_removed_obs_group_importance())
def dataworth_test():
import os
import numpy as np
from pyemu import Schur,Cov
w_dir = os.path.join("..","verification","Freyberg")
forecasts = ["travel_time","sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
#sc.pst.observation_data.loc[sc.pst.nnz_obs_names[3:],"weight"] = 0.0
base_obs = sc.pst.nnz_obs_names
zw_names = [name for name in sc.pst.zero_weight_obs_names if name not in forecasts ]
#print(sc.get_removed_obs_importance(obslist_dict={"test":zw_names}))
oname = "or00c00_0"
names = {"test":oname}
added = sc.get_added_obs_importance(base_obslist=base_obs,
obslist_dict=names,reset_zero_weight=1.0)
sc.pst.observation_data.loc[oname,"weight"] = 1.0
#sc.reset_obscov()
cov = Cov.from_observation_data(sc.pst)
scm = Schur(jco=os.path.join(w_dir, "freyberg.jcb"), forecasts=forecasts, verbose=True,
obscov=cov)
scm.pst.observation_data.loc[oname, "weight"] = 1.0
removed = sc.get_removed_obs_importance(obslist_dict=names)
for fname in forecasts:
diff = np.abs(removed.loc["test",fname] - added.loc["base",fname])
print("add,{0},{1},{2}".format(fname, added.loc["test", fname], added.loc["base", fname]))
print("rem,{0},{1},{2}".format(fname, removed.loc["test", fname], removed.loc["base", fname]))
assert diff < 0.01,"{0},{1},{2}".format(fname,removed.loc["test",fname],added.loc["base",fname])
names = {"test1":oname,"test2":["or00c00_1","or00c00_2"]}
scm.pst.observation_data.loc[oname, "weight"] = 0.0
scm.next_most_important_added_obs(forecast="travel_time",obslist_dict=names,
base_obslist=scm.pst.nnz_obs_names,reset_zero_weight=1.0)
def schur_test():
import os
import numpy as np
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
pst = Pst(os.path.join(w_dir,"pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc"))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_parameter_summary(include_map=True))
print(sc.get_forecast_summary(include_map=True))
print(sc.get_removed_obs_importance(reset_zero_weight=True))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
sigma_range=6.0)
cov = Cov.from_parameter_data(pst,sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0,scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
pst.parameter_data.loc[:,"offset"] = 100.0
cov = Cov.from_parameter_data(pst)
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
pst=pst,
forecasts=forecasts,
sigma_range=6.0, scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0
cov = Cov.from_parameter_data(pst,scale_offset=False,sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def errvar_test_nonpest():
import numpy as np
from pyemu import ErrVar, Matrix, Cov
#non-pest
pnames = ["p1","p2","p3"]
onames = ["o1","o2","o3","o4"]
npar = len(pnames)
nobs = len(onames)
j_arr = np.random.random((nobs,npar))
jco = Matrix(x=j_arr,row_names=onames,col_names=pnames)
parcov = Cov(x=np.eye(npar),names=pnames)
obscov = Cov(x=np.eye(nobs),names=onames)
forecasts = "o2"
omitted = "p3"
e = ErrVar(jco=jco,parcov=parcov,obscov=obscov,forecasts=forecasts,
omitted_parameters=omitted)
svs = [0,1,2,3,4,5]
print(e.get_errvar_dataframe(svs))
def covariance_matrix(self, x, y, names=None, cov=None):
"""build a pyemu.Cov instance from GeoStruct
Parameters
----------
x : iterable of floats
x locations
y : iterable of floats
y locations
names : iterable of str (optional)
names of location. If None, generic names will be used
cov : pyemu.Cov instance (optional)
an existing Cov instance to add contribution to
Returns
-------
pyemu.Cov
"""
if not isinstance(x, np.ndarray):
x = np.array(x)
if not isinstance(y, np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names), len(names)))
np.fill_diagonal(c, self.nugget)
cov = Cov(x=c, names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names), 1))
c += self.nugget
cont = Cov(x=c, names=names, isdiagonal=True)
cov += cont
else:
raise Exception("GeoStruct.covariance_matrix() requires either " +
"names or cov arg")
for v in self.variograms:
v.covariance_matrix(x, y, cov=cov)
return cov
def covariance_matrix(self, x, y, names=None, cov=None):
if not isinstance(x, np.ndarray):
x = np.array(x)
if not isinstance(y, np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names), len(names)))
np.fill_diagonal(c, self.contribution)
cov = Cov(x=c, names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names), 1)) + self.contribution
cont = Cov(x=c, names=names, isdiagonal=True)
cov += cont
else:
raise Exception("Vario2d.covariance_matrix() requires either" +
"names or cov arg")
for i1, (n1, x1, y1) in enumerate(zip(names, x, y)):
dx = x1 - x[i1 + 1:]
dy = y1 - y[i1 + 1:]
dxx = (dx * self.rotation_coefs[0]) +\
(dy * self.rotation_coefs[1])
dyy = ((dx * self.rotation_coefs[2]) +\
(dy * self.rotation_coefs[3])) *\
self.anisotropy
h = np.sqrt(dxx * dxx + dyy * dyy)
h[h < 0.0] = 0.0
cov.x[i1, i1 + 1:] += self.h_function(h)
for i in range(len(names)):
cov.x[i + 1:, i] = cov.x[i, i + 1:]
return cov
def covariance_matrix(self,x,y,names=None,cov=None):
if not isinstance(x,np.ndarray):
x = np.array(x)
if not isinstance(y,np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names),len(names)))
np.fill_diagonal(c,self.contribution)
cov = Cov(x=c,names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names),1)) + self.contribution
cont = Cov(x=c,names=names,isdiagonal=True)
cov += cont
else:
raise Exception("Vario2d.covariance_matrix() requires either" +
"names or cov arg")
for i1,(n1,x1,y1) in enumerate(zip(names,x,y)):
dx = x1 - x[i1+1:]
dy = y1 - y[i1+1:]
dxx = (dx * self.rotation_coefs[0]) +\
(dy * self.rotation_coefs[1])
dyy = ((dx * self.rotation_coefs[2]) +\
(dy * self.rotation_coefs[3])) *\
self.anisotropy
h = np.sqrt(dxx*dxx + dyy*dyy)
h[h<0.0] = 0.0
cov.x[i1,i1+1:] += self.h_function(h)
for i in range(len(names)):
cov.x[i+1:,i] = cov.x[i,i+1:]
return cov
def covariance_matrix(self, x, y, names=None, cov=None):
if not isinstance(x, np.ndarray):
x = np.array(x)
if not isinstance(y, np.ndarray):
y = np.array(y)
assert x.shape[0] == y.shape[0]
if names is not None:
assert x.shape[0] == len(names)
c = np.zeros((len(names), len(names)))
np.fill_diagonal(c, self.nugget)
cov = Cov(x=c, names=names)
elif cov is not None:
assert cov.shape[0] == x.shape[0]
names = cov.row_names
c = np.zeros((len(names), 1)) + self.nugget
cont = Cov(x=c, names=names, isdiagonal=True)
cov += cont
else:
raise Exception("GeoStruct.covariance_matrix() requires either" +
"names or cov arg")
for v in self.variograms:
v.covariance_matrix(x, y, cov=cov)
return cov
def la_test_io():
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_binary(os.path.join("temp", "pest.bin.cov"))
cov.to_ascii(os.path.join("temp", "pest.txt.cov"))
sc_bin = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp", "pest.bin.cov"))
sc_ascii = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp", "pest.txt.cov"))
def la_test_io():
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
pst = Pst(os.path.join(w_dir,"pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_binary(os.path.join("temp","pest.bin.cov"))
cov.to_ascii(os.path.join("temp","pest.txt.cov"))
sc_bin = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp","pest.bin.cov"))
sc_ascii = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp","pest.txt.cov"))
def diagonal_cov_draw_test():
import os
import numpy as np
from pyemu import MonteCarlo, Cov, Pst
jco = os.path.join("pst", "pest.jcb")
pst = Pst(jco.replace(".jcb", ".pst"))
mc = MonteCarlo(jco=jco, pst=pst)
num_reals = 10
mc.draw(num_reals, obs=True)
print(mc.obsensemble)
pe1 = mc.parensemble.copy()
cov = Cov(x=mc.parcov.as_2d, names=mc.parcov.row_names)
#print(type(cov))
mc = MonteCarlo(jco=jco, pst=pst)
mc.parensemble.reseed()
mc.draw(num_reals, cov=cov)
pe2 = mc.parensemble
def from_dataframe_test():
import os
import numpy as np
import pandas as pd
from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst, Cov
jco = os.path.join("pst","pest.jcb")
pst = jco.replace(".jcb",".pst")
mc = MonteCarlo(jco=jco,pst=pst)
names = ["par_{0}".format(_) for _ in range(10)]
df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names)
mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst)
print(mc.parensemble.shape)
mc.project_parensemble()
mc.parensemble.to_csv(os.path.join("temp","test.csv"))
pstc = Pst(pst)
par = pstc.parameter_data
par.sort_values(by="parnme",ascending=False,inplace=True)
cov = Cov.from_parameter_data(pstc)
pe = ParameterEnsemble.from_gaussian_draw(pst=mc.pst,cov=cov)
