def covariance_matrix(self, x, y, names=None, cov=None): """build a pyemu.Cov instance from Vario2d Parameters ---------- x : iterable of floats x locations y : iterable of floats y locations names : iterable of str (optional) names of location. If None, generic names will be used cov : pyemu.Cov instance (optional) an existing Cov instance to add contribution to Returns ------- pyemu.Cov """ if not isinstance(x, np.ndarray): x = np.array(x) if not isinstance(y, np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names), len(names))) np.fill_diagonal(c, self.contribution) cov = Cov(x=c, names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names), 1)) + self.contribution cont = Cov(x=c, names=names, isdiagonal=True) cov += cont else: raise Exception("Vario2d.covariance_matrix() requires either" + "names or cov arg") for i1, (n1, x1, y1) in enumerate(zip(names, x, y)): dx = x1 - x[i1 + 1:] dy = y1 - y[i1 + 1:] dxx = (dx * self.rotation_coefs[0]) +\ (dy * self.rotation_coefs[1]) dyy = ((dx * self.rotation_coefs[2]) +\ (dy * self.rotation_coefs[3])) *\ self.anisotropy h = np.sqrt(dxx * dxx + dyy * dyy) h[h < 0.0] = 0.0 h = self.h_function(h) if np.any(np.isnan(h)): raise Exception("nans in h for i1 {0}".format(i1)) cov.x[i1, i1 + 1:] += h for i in range(len(names)): cov.x[i + 1:, i] = cov.x[i, i + 1:] return cov
def schur_test_nonpest(): import numpy as np from pyemu import Matrix, Cov, Schur, Jco #non-pest pnames = ["p1", "p2", "p3"] onames = ["o1", "o2", "o3", "o4"] npar = len(pnames) nobs = len(onames) j_arr = np.random.random((nobs, npar)) jco = Jco(x=j_arr, row_names=onames, col_names=pnames) parcov = Cov(x=np.eye(npar), names=pnames) obscov = Cov(x=np.eye(nobs), names=onames) forecasts = "o2" s = Schur(jco=jco, parcov=parcov, obscov=obscov, forecasts=forecasts) print(s.get_parameter_summary()) print(s.get_forecast_summary()) #this should fail passed = False try: print(s.get_par_group_contribution()) passed = True except Exception as e: print(str(e)) if passed: raise Exception("should have failed") #this should fail passed = False try: print(s.get_removed_obs_group_importance()) passed = True except Exception as e: print(str(e)) if passed: raise Exception("should have failed") print(s.get_par_contribution({"group1": ["p1", "p3"]})) print(s.get_removed_obs_importance({"group1": ["o1", "o3"]})) print(s.pandas) forecasts = Matrix(x=np.random.random((1, npar)), row_names=[forecasts], col_names=pnames) sc = Schur(jco=jco, forecasts=forecasts.T, parcov=parcov, obscov=obscov) ffile = os.path.join("temp", "forecasts.jcb") forecasts.to_binary(ffile) sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov)
def covariance_matrix(self,x,y,names=None,cov=None): """build a pyemu.Cov instance from Vario2d Parameters ---------- x : iterable of floats x locations y : iterable of floats y locations names : iterable of str (optional) names of location. If None, generic names will be used cov : pyemu.Cov instance (optional) an existing Cov instance to add contribution to Returns ------- pyemu.Cov """ if not isinstance(x,np.ndarray): x = np.array(x) if not isinstance(y,np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names),len(names))) np.fill_diagonal(c,self.contribution) cov = Cov(x=c,names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names),1)) + self.contribution cont = Cov(x=c,names=names,isdiagonal=True) cov += cont else: raise Exception("Vario2d.covariance_matrix() requires either" + "names or cov arg") for i1,(n1,x1,y1) in enumerate(zip(names,x,y)): dx = x1 - x[i1+1:] dy = y1 - y[i1+1:] dxx = (dx * self.rotation_coefs[0]) +\ (dy * self.rotation_coefs[1]) dyy = ((dx * self.rotation_coefs[2]) +\ (dy * self.rotation_coefs[3])) *\ self.anisotropy h = np.sqrt(dxx*dxx + dyy*dyy) h[h<0.0] = 0.0 cov.x[i1,i1+1:] += self.h_function(h) for i in range(len(names)): cov.x[i+1:,i] = cov.x[i,i+1:] return cov
def schur_test(): import os import numpy as np from pyemu import Schur, Cov, Pst w_dir = os.path.join("..", "verification", "henry") forecasts = ["pd_ten", "c_obs10_2"] pst = Pst(os.path.join(w_dir, "pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None) cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc")) sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, parcov=cov2) print(sc.prior_forecast) print(sc.posterior_forecast) print(sc.get_par_group_contribution()) df = sc.get_par_group_contribution(include_prior_results=True) levels = list(df.columns.levels[1]) assert "prior" in levels, levels assert "post" in levels, levels print(sc.get_parameter_summary(include_map=True)) print(sc.get_forecast_summary(include_map=True)) print(sc.get_removed_obs_importance(reset_zero_weight=True)) sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, sigma_range=6.0) cov = Cov.from_parameter_data(pst, sigma_range=6.0) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0 sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, sigma_range=6.0, scale_offset=False) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0 pst.parameter_data.loc[:, "offset"] = 100.0 cov = Cov.from_parameter_data(pst) sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), pst=pst, forecasts=forecasts, sigma_range=6.0, scale_offset=False) assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0 cov = Cov.from_parameter_data(pst, scale_offset=False, sigma_range=6.0) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def gaussian_draw_test(): import os import numpy as np from pyemu import MonteCarlo,Cov,ParameterEnsemble from datetime import datetime jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) num_reals = 100 start = datetime.now() mc.draw(num_reals=num_reals,how="gaussian") print(mc.parensemble.head()) print(datetime.now() - start) vals = mc.pst.parameter_data.parval1.values cov = Cov.from_parameter_data(mc.pst) start = datetime.now() val_array = np.random.multivariate_normal(vals, cov.as_2d,num_reals) print(datetime.now() - start) start = datetime.now() pe = ParameterEnsemble.from_gaussian_draw(mc.pst,cov,num_reals=num_reals) pet = pe._transform(inplace=False) pe = pet._back_transform(inplace=False) print(datetime.now() - start) print(mc.parensemble.head()) print(pe.head())
def schur_test(): import os from pyemu import Schur, Cov, Pst w_dir = os.path.join("..", "verification", "henry") forecasts = ["pd_ten", "c_obs10_2"] pst = Pst(os.path.join(w_dir, "pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None) cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc")) sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, parcov=cov2) print(sc.prior_forecast) print(sc.posterior_forecast) print(sc.get_par_group_contribution()) print(sc.get_removed_obs_group_importance())
def mc_test(): import os import numpy as np from pyemu import MonteCarlo, Cov jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") out_dir = os.path.join("mc") if not os.path.exists(out_dir): os.mkdir(out_dir) #write testing mc = MonteCarlo(jco=jco,verbose=True,sigma_range=6) cov = Cov.from_parameter_data(mc.pst,sigma_range=6) assert np.abs((mc.parcov.x - cov.x).sum()) == 0.0 mc.draw(10,obs=True) mc.write_psts(os.path.join("temp","real_")) mc.parensemble.to_parfiles(os.path.join("mc","real_")) mc = MonteCarlo(jco=jco,verbose=True) mc.draw(10,obs=True) print("prior ensemble variance:", np.var(mc.parensemble.loc[:,"mult1"])) projected_en = mc.project_parensemble(inplace=False) print("projected ensemble variance:", np.var(projected_en.loc[:,"mult1"])) import pyemu sc = pyemu.Schur(jco=jco) mc = MonteCarlo(pst=pst,parcov=sc.posterior_parameter,verbose=True) mc.draw(10) print("posterior ensemble variance:", np.var(mc.parensemble.loc[:,"mult1"]))
def schur_test(): import os from pyemu import Schur, Cov, Pst w_dir = os.path.join("..","..","verification","henry") forecasts = ["pd_ten","c_obs10_2"] pst = Pst(os.path.join(w_dir,"pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None) cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc")) sc = Schur(jco=os.path.join(w_dir,"pest.jcb"), forecasts=forecasts, parcov=cov2) print(sc.prior_forecast) print(sc.posterior_forecast) print(sc.get_par_group_contribution()) print(sc.get_removed_obs_group_importance())
def dataworth_test(): import os import numpy as np from pyemu import Schur,Cov w_dir = os.path.join("..","verification","Freyberg") forecasts = ["travel_time","sw_gw_0","sw_gw_1"] sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True) #sc.pst.observation_data.loc[sc.pst.nnz_obs_names[3:],"weight"] = 0.0 base_obs = sc.pst.nnz_obs_names zw_names = [name for name in sc.pst.zero_weight_obs_names if name not in forecasts ] #print(sc.get_removed_obs_importance(obslist_dict={"test":zw_names})) oname = "or00c00_0" names = {"test":oname} added = sc.get_added_obs_importance(base_obslist=base_obs, obslist_dict=names,reset_zero_weight=1.0) sc.pst.observation_data.loc[oname,"weight"] = 1.0 #sc.reset_obscov() cov = Cov.from_observation_data(sc.pst) scm = Schur(jco=os.path.join(w_dir, "freyberg.jcb"), forecasts=forecasts, verbose=True, obscov=cov) scm.pst.observation_data.loc[oname, "weight"] = 1.0 removed = sc.get_removed_obs_importance(obslist_dict=names) for fname in forecasts: diff = np.abs(removed.loc["test",fname] - added.loc["base",fname]) print("add,{0},{1},{2}".format(fname, added.loc["test", fname], added.loc["base", fname])) print("rem,{0},{1},{2}".format(fname, removed.loc["test", fname], removed.loc["base", fname])) assert diff < 0.01,"{0},{1},{2}".format(fname,removed.loc["test",fname],added.loc["base",fname]) names = {"test1":oname,"test2":["or00c00_1","or00c00_2"]} scm.pst.observation_data.loc[oname, "weight"] = 0.0 scm.next_most_important_added_obs(forecast="travel_time",obslist_dict=names, base_obslist=scm.pst.nnz_obs_names,reset_zero_weight=1.0)
def schur_test(): import os import numpy as np from pyemu import Schur, Cov, Pst w_dir = os.path.join("..","verification","henry") forecasts = ["pd_ten","c_obs10_2"] pst = Pst(os.path.join(w_dir,"pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None) cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc")) sc = Schur(jco=os.path.join(w_dir,"pest.jcb"), forecasts=forecasts, parcov=cov2) print(sc.prior_forecast) print(sc.posterior_forecast) print(sc.get_par_group_contribution()) print(sc.get_parameter_summary(include_map=True)) print(sc.get_forecast_summary(include_map=True)) print(sc.get_removed_obs_importance(reset_zero_weight=True)) sc = Schur(jco=os.path.join(w_dir,"pest.jcb"), forecasts=forecasts, sigma_range=6.0) cov = Cov.from_parameter_data(pst,sigma_range=6.0) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0 sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, sigma_range=6.0,scale_offset=False) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0 pst.parameter_data.loc[:,"offset"] = 100.0 cov = Cov.from_parameter_data(pst) sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), pst=pst, forecasts=forecasts, sigma_range=6.0, scale_offset=False) assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0 cov = Cov.from_parameter_data(pst,scale_offset=False,sigma_range=6.0) assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def errvar_test_nonpest(): import numpy as np from pyemu import ErrVar, Matrix, Cov #non-pest pnames = ["p1","p2","p3"] onames = ["o1","o2","o3","o4"] npar = len(pnames) nobs = len(onames) j_arr = np.random.random((nobs,npar)) jco = Matrix(x=j_arr,row_names=onames,col_names=pnames) parcov = Cov(x=np.eye(npar),names=pnames) obscov = Cov(x=np.eye(nobs),names=onames) forecasts = "o2" omitted = "p3" e = ErrVar(jco=jco,parcov=parcov,obscov=obscov,forecasts=forecasts, omitted_parameters=omitted) svs = [0,1,2,3,4,5] print(e.get_errvar_dataframe(svs))
def covariance_matrix(self, x, y, names=None, cov=None): """build a pyemu.Cov instance from GeoStruct Parameters ---------- x : iterable of floats x locations y : iterable of floats y locations names : iterable of str (optional) names of location. If None, generic names will be used cov : pyemu.Cov instance (optional) an existing Cov instance to add contribution to Returns ------- pyemu.Cov """ if not isinstance(x, np.ndarray): x = np.array(x) if not isinstance(y, np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names), len(names))) np.fill_diagonal(c, self.nugget) cov = Cov(x=c, names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names), 1)) c += self.nugget cont = Cov(x=c, names=names, isdiagonal=True) cov += cont else: raise Exception("GeoStruct.covariance_matrix() requires either " + "names or cov arg") for v in self.variograms: v.covariance_matrix(x, y, cov=cov) return cov
def covariance_matrix(self, x, y, names=None, cov=None): if not isinstance(x, np.ndarray): x = np.array(x) if not isinstance(y, np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names), len(names))) np.fill_diagonal(c, self.contribution) cov = Cov(x=c, names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names), 1)) + self.contribution cont = Cov(x=c, names=names, isdiagonal=True) cov += cont else: raise Exception("Vario2d.covariance_matrix() requires either" + "names or cov arg") for i1, (n1, x1, y1) in enumerate(zip(names, x, y)): dx = x1 - x[i1 + 1:] dy = y1 - y[i1 + 1:] dxx = (dx * self.rotation_coefs[0]) +\ (dy * self.rotation_coefs[1]) dyy = ((dx * self.rotation_coefs[2]) +\ (dy * self.rotation_coefs[3])) *\ self.anisotropy h = np.sqrt(dxx * dxx + dyy * dyy) h[h < 0.0] = 0.0 cov.x[i1, i1 + 1:] += self.h_function(h) for i in range(len(names)): cov.x[i + 1:, i] = cov.x[i, i + 1:] return cov
def covariance_matrix(self,x,y,names=None,cov=None): if not isinstance(x,np.ndarray): x = np.array(x) if not isinstance(y,np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names),len(names))) np.fill_diagonal(c,self.contribution) cov = Cov(x=c,names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names),1)) + self.contribution cont = Cov(x=c,names=names,isdiagonal=True) cov += cont else: raise Exception("Vario2d.covariance_matrix() requires either" + "names or cov arg") for i1,(n1,x1,y1) in enumerate(zip(names,x,y)): dx = x1 - x[i1+1:] dy = y1 - y[i1+1:] dxx = (dx * self.rotation_coefs[0]) +\ (dy * self.rotation_coefs[1]) dyy = ((dx * self.rotation_coefs[2]) +\ (dy * self.rotation_coefs[3])) *\ self.anisotropy h = np.sqrt(dxx*dxx + dyy*dyy) h[h<0.0] = 0.0 cov.x[i1,i1+1:] += self.h_function(h) for i in range(len(names)): cov.x[i+1:,i] = cov.x[i,i+1:] return cov
def covariance_matrix(self, x, y, names=None, cov=None): if not isinstance(x, np.ndarray): x = np.array(x) if not isinstance(y, np.ndarray): y = np.array(y) assert x.shape[0] == y.shape[0] if names is not None: assert x.shape[0] == len(names) c = np.zeros((len(names), len(names))) np.fill_diagonal(c, self.nugget) cov = Cov(x=c, names=names) elif cov is not None: assert cov.shape[0] == x.shape[0] names = cov.row_names c = np.zeros((len(names), 1)) + self.nugget cont = Cov(x=c, names=names, isdiagonal=True) cov += cont else: raise Exception("GeoStruct.covariance_matrix() requires either" + "names or cov arg") for v in self.variograms: v.covariance_matrix(x, y, cov=cov) return cov
def la_test_io(): from pyemu import Schur, Cov, Pst w_dir = os.path.join("..", "verification", "henry") forecasts = ["pd_ten", "c_obs10_2"] pst = Pst(os.path.join(w_dir, "pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_binary(os.path.join("temp", "pest.bin.cov")) cov.to_ascii(os.path.join("temp", "pest.txt.cov")) sc_bin = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, parcov=os.path.join("temp", "pest.bin.cov")) sc_ascii = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts, parcov=os.path.join("temp", "pest.txt.cov"))
def la_test_io(): from pyemu import Schur, Cov, Pst w_dir = os.path.join("..","verification","henry") forecasts = ["pd_ten","c_obs10_2"] pst = Pst(os.path.join(w_dir,"pest.pst")) cov = Cov.from_parameter_data(pst) cov.to_binary(os.path.join("temp","pest.bin.cov")) cov.to_ascii(os.path.join("temp","pest.txt.cov")) sc_bin = Schur(jco=os.path.join(w_dir,"pest.jcb"), forecasts=forecasts, parcov=os.path.join("temp","pest.bin.cov")) sc_ascii = Schur(jco=os.path.join(w_dir,"pest.jcb"), forecasts=forecasts, parcov=os.path.join("temp","pest.txt.cov"))
def diagonal_cov_draw_test(): import os import numpy as np from pyemu import MonteCarlo, Cov, Pst jco = os.path.join("pst", "pest.jcb") pst = Pst(jco.replace(".jcb", ".pst")) mc = MonteCarlo(jco=jco, pst=pst) num_reals = 10 mc.draw(num_reals, obs=True) print(mc.obsensemble) pe1 = mc.parensemble.copy() cov = Cov(x=mc.parcov.as_2d, names=mc.parcov.row_names) #print(type(cov)) mc = MonteCarlo(jco=jco, pst=pst) mc.parensemble.reseed() mc.draw(num_reals, cov=cov) pe2 = mc.parensemble
def from_dataframe_test(): import os import numpy as np import pandas as pd from pyemu import MonteCarlo,Ensemble,ParameterEnsemble,Pst, Cov jco = os.path.join("pst","pest.jcb") pst = jco.replace(".jcb",".pst") mc = MonteCarlo(jco=jco,pst=pst) names = ["par_{0}".format(_) for _ in range(10)] df = pd.DataFrame(np.random.random((10,mc.pst.npar)),columns=mc.pst.par_names) mc.parensemble = ParameterEnsemble.from_dataframe(df=df,pst=mc.pst) print(mc.parensemble.shape) mc.project_parensemble() mc.parensemble.to_csv(os.path.join("temp","test.csv")) pstc = Pst(pst) par = pstc.parameter_data par.sort_values(by="parnme",ascending=False,inplace=True) cov = Cov.from_parameter_data(pstc) pe = ParameterEnsemble.from_gaussian_draw(pst=mc.pst,cov=cov)