def from_parbounds(self, pst_file):
if not pst_file.endswith(".pst"):
pst_file += ".pst"
import pst_handler as phand
pst = phand.pst(pst_file)
visited = [False] * len(self.names)
for i, row in pst.parameter_data.iterrows():
if row["parnme"] in self.names:
idx = self.names.index(row["parnme"])
t = row["partrans"]
if t in ["fixed", "tied"]:
raise Exception("fixed or tied parameter: " +
row["parnme"])
lb, ub = row["parlbnd"], row["parubnd"]
if t == "log":
var = ((np.log10(ub) - np.log10(lb)) / 4.0)**2
else:
var = ((ub - lb) / 4.0)**2
self.x[idx, idx] = var
visited[idx] = True
if False in visited:
for name, visit in zip(self.names, visited):
if not visit:
print 'entry not found for name:', name
raise Exception(
'error loading uncertainty from observations weights')
def from_obsweights(self,pst_file):
if not pst_file.endswith(".pst"):
pst_file += ".pst"
import pst_handler as phand
pst = phand.pst(pst_file)
visited = [False] * len(self.names)
for i,row in pst.observation_data.iterrows():
if row["obsnme"] in self.names:
idx = self.names.index(row["obsnme"])
w = row["weight"]
if w == 0.0:
#raise Exception("zero weight observation: "+row["obsnme"])
print "resetting weight for",row["obsnme"],"from 0.0 to 1.0e-30"
w = 1.0e-30
self.x[idx,idx] = (1.0/w)**2
visited[idx] = True
if False in visited:
for name,visit in zip(self.names,visited):
if not visit:
print 'entry not found for name:',name
raise Exception('error loading uncertainty from observations weights')
def from_obsweights(self, pst_file):
if not pst_file.endswith(".pst"):
pst_file += ".pst"
import pst_handler as phand
pst = phand.pst(pst_file)
visited = [False] * len(self.names)
for i, row in pst.observation_data.iterrows():
if row["obsnme"] in self.names:
idx = self.names.index(row["obsnme"])
w = row["weight"]
if w == 0.0:
#raise Exception("zero weight observation: "+row["obsnme"])
print "resetting weight for", row[
"obsnme"], "from 0.0 to 1.0e-30"
w = 1.0e-30
self.x[idx, idx] = (1.0 / w)**2
visited[idx] = True
if False in visited:
for name, visit in zip(self.names, visited):
if not visit:
print 'entry not found for name:', name
raise Exception(
'error loading uncertainty from observations weights')
def from_parbounds(self,pst_file):
if not pst_file.endswith(".pst"):
pst_file += ".pst"
import pst_handler as phand
pst = phand.pst(pst_file)
visited = [False] * len(self.names)
for i,row in pst.parameter_data.iterrows():
if row["parnme"] in self.names:
idx = self.names.index(row["parnme"])
t = row["partrans"]
if t in ["fixed","tied"]:
raise Exception("fixed or tied parameter: "+row["parnme"])
lb,ub = row["parlbnd"],row["parubnd"]
if t == "log":
var = ((np.log10(ub) - np.log10(lb))/4.0)**2
else:
var = ((ub - lb)/4.0)**2
self.x[idx,idx] = var
visited[idx] = True
if False in visited:
for name,visit in zip(self.names,visited):
if not visit:
print 'entry not found for name:',name
raise Exception('error loading uncertainty from observations weights')
import os
from copy import deepcopy
import numpy as np
import pandas
import pst_handler
from pst_handler import pst
p = pst()
p.read_pst('umd02.pst')
p.reconcile_prior_2_pars()
out_dir = 'tests\\'
test_count = 1
#--find paired processed/raw obs groups
processed, raw = [], []
obs_grps = list(p.observation_groups['obgnme'].values)
for ogp in obs_grps:
if ogp + '_r' in obs_grps:
processed.append(ogp)
raw.append(ogp + '_r')
#--seperate processed and raw
sel1, sel2 = [], []
bf_obsname = []
pred_groups = []
for ogp in p.observation_groups['obgnme'].values:
if ogp in processed:
sel1.append(False)
sel2.append(True)
elif ogp in raw:
sel1.append(True)
sel2.append(False)
else:
'''
param = std
etsx = 0.25m
petm = 0.25
pseg = 0.0 (all fixed)
nex = 0.1
cond = 100
'''
std_dict = {
'etsx': 0.25,
'petm': 0.25,
'nex': 0.1,
'cond': 100,
'pseg': 1.0e-10
}
p = pst(filename='..\\umd02.pst')
f_unc_sc = open('umd02_scaled.unc', 'w', 0)
f_unc_sc.write('START STANDARD_DEVIATION\n')
for pname in p.parameter_data.parnme:
f_unc_sc.write(' {0:20s} {1:15.7G}\n'.format(pname, 1.0))
f_unc_sc.write('END STANDARD_DEVIATION\n\n')
f_unc_sc.close()
f_unc = open('umd02.unc', 'w', 0)
for pargp in p.parameter_groups.pargpnme:
if pargp in std_dict.keys():
f_unc.write('START STANDARD_DEVIATION\n')
params = p.parameter_data[p.parameter_data.pargp == pargp]
for i in params.index:
par = params.ix[i]
#--uncorrelated uncertainty by group
'''
param = std
etsx = 0.25m
petm = 0.25
pseg = 0.0 (all fixed)
nex = 0.1
cond = 100
'''
std_dict = {'etsx':0.25,'petm':0.25,'nex':0.1,'cond':100,'pseg':1.0e-10}
p = pst(filename='..\\umd02.pst')
f_unc_sc = open('umd02_scaled.unc','w',0)
f_unc_sc.write('START STANDARD_DEVIATION\n')
for pname in p.parameter_data.parnme:
f_unc_sc.write(' {0:20s} {1:15.7G}\n'.format(pname,1.0))
f_unc_sc.write('END STANDARD_DEVIATION\n\n')
f_unc_sc.close()
f_unc = open('umd02.unc','w',0)
for pargp in p.parameter_groups.pargpnme:
if pargp in std_dict.keys():
f_unc.write('START STANDARD_DEVIATION\n')
params = p.parameter_data[p.parameter_data.pargp==pargp]
for i in params.index:
par = params.ix[i]
def load_pst(self):
return phand.pst(self.case + ".pst")
import numpy as np
import pylab
import pandas
import pst_handler as ph
pst = ph.pst('umd02_processed_head.pst')
pst.observation_data.index = pst.observation_data.obsnme
df_file = '\\\\IGSBAMEWMS152\\Home\jwhite\MiamiDade\\umd02_la_master\\sliced\\i64predunc1_unscaled.csv'
df = pandas.read_csv(df_file, sep='|', index_col=0)
pred_obs = pst.observation_data.ix[df.index.values].dropna()
print pst.observation_data.obsval.shape, df.shape
df = np.abs(df.div(pred_obs.obsval, axis=0) * 100.0)
col_strings = df.columns
rawpro, bf, prepost = [], [], []
for c in col_strings:
if 'raw' in c:
rawpro.append('raw')
else:
rawpro.append('processed')
if 'bf' in c:
bf.append('with_bf')
else:
bf.append('no_bf')
if 'pre' in c:
prepost.append('pre-cal')
else:
prepost.append('post-cal')
tups = zip(*(rawpro, bf, prepost))
df.columns = pandas.MultiIndex.from_tuples(tups,
def main(in_filename,verbose=False):
#--load the parms from the infile
f = open(in_filename,'r')
pst_name = f.readline().strip()
jco_name = pst_name.replace('.pst','.jco')
ref_var = float(f.readline().strip())
unc_filename = f.readline().strip()
pv_list_file = f.readline().strip()
sing_file = f.readline().strip()
try:
struct_file = f.readline().strip()
except:
struct_file = None
f.close()
#pv_list_file = 'biweekly\\biweekly_scaled.list'
pv_dict = {}
pvec = mhand.matrix()
f = open(pv_list_file,'r')
for line in f:
if not line.startswith('#'):
raw = line.strip().split()
print 'loading pred vector',raw[0]
pvec.from_ascii(raw[0])
pv_dict[raw[1]] = copy.deepcopy(pvec.x)
f.close()
#sing_vals = np.loadtxt('singular_values.dat',dtype=np.int32)
sing_vals = np.loadtxt(sing_file,dtype=np.int32)
#sing_vals = [2]
print 'loading pest control file',pst_name
pst = phand.pst(pst_name)
print 'loading jco file',jco_name
jco = mhand.matrix()
jco.from_binary(jco_name)
print 'dropping zero-weight obs from jco'
drop_row_names = []
nz_weights,nz_names = [],[]
for name,weight in zip(pst.observation_data.obsnme,pst.observation_data.weight):
if weight == 0.0:
drop_row_names.append(name)
else:
nz_weights.append(weight)
nz_names.append(name)
jco.drop(drop_row_names)
print 'building obs cov matrix'
obs_unc = mhand.uncert(nz_names)
obs_unc.from_uncfile('simple_obs.unc')
print 'inverting obs cov matrix = q'
print 'and square root of q'
#q = np.linalg.inv(obs_unc.x)
#qhalf= linalg.sqrtm(q).real
u,s,vt = linalg.svd(obs_unc.x)
for i in range(u.shape[0]):
u[:,i] *= s[i]**(-0.5)
qhalf = np.dot(u,vt)
print 'build q^(1/2)X'
qX = np.dot(qhalf,jco.x)
if verbose:
np.savetxt('qX_py.mat',qX,fmt='%15.6E')
np.savetxt('qhalf_py.mat',qhalf,fmt='%15.6E')
np.savetxt('X_py.mat',jco.x,fmt='%15.6E')
jco_struct = None
spv_dict = None
if struct_file:
print 'loading structural parameter list'
spar_names,idxs = [],[]
#f = open('struct_pars.dat','r')
f = open(struct_file,'r')
for line in f:
name = line.strip()
if name in jco.col_names:
spar_names.append(name)
idxs.append(jco.col_names.index(name))
f.close()
print 'extracting omitted elements from q^(1/2)X'
jco_struct_q = qX[:,idxs]
jco_struct = jco.x[:,idxs]
qX = np.delete(qX,idxs,1)
X = np.delete(jco.x,idxs,1)
print 'forming omitted pred vectors'
spv_dict = {}
for out_name,pv in pv_dict.iteritems():
spv = np.atleast_2d(pv[idxs])
pv = np.atleast_2d(np.delete(pv,idxs)).transpose()
pv_dict[out_name] = pv
spv_dict[out_name] = spv
if 'halfscaled' in in_filename:
raise
print 'loading param uncert file for unscaled struct parameters'
unc = mhand.uncert(spar_names)
unc.from_uncfile('simple.unc')
#unc_struct = unc.x[idxs:idxs]
print unc_struct.shape
par_cov = None
struct_cov = None
if 'scaled' not in pst_name:
print 'building par cov matrix'
nspar_names = []
for name in jco.col_names:
if name not in spar_names:
nspar_names.append(name)
par_unc = mhand.uncert(nspar_names)
par_unc.from_uncfile('simple.unc')
if verbose:
np.savetxt('pycp.mat',par_unc.x,fmt='%15.6E')
print 'building structual par cov matrix'
struct_unc = mhand.uncert(spar_names)
struct_unc.from_uncfile('simple.unc')
par_cov = par_unc.x
struct_cov = struct_unc.x
pv = pvar.predvar(qX,pv_dict,sing_vals,verbose=verbose,jco_struct=jco_struct_q,\
pred_vectors_struct=spv_dict,struct_cov=struct_cov,par_cov=par_cov)
#pv = pvar.predvar(X,pv_dict,sing_vals,verbose=verbose,jco_struct=jco_struct,\
# pred_vectors_struct=spv_dict,struct_cov=struct_cov,par_cov=par_cov,obs_cov=obs_unc.x)
pv.calc()
def main(in_filename, verbose=False):
#--load the parms from the infile
f = open(in_filename, 'r')
pst_name = f.readline().strip()
jco_name = pst_name.replace('.pst', '.jco')
ref_var = float(f.readline().strip())
unc_filename = f.readline().strip()
pv_list_file = f.readline().strip()
sing_file = f.readline().strip()
try:
struct_file = f.readline().strip()
except:
struct_file = None
f.close()
#pv_list_file = 'biweekly\\biweekly_scaled.list'
pv_dict = {}
pvec = mhand.matrix()
f = open(pv_list_file, 'r')
for line in f:
if not line.startswith('#'):
raw = line.strip().split()
print 'loading pred vector', raw[0]
pvec.from_ascii(raw[0])
pv_dict[raw[1]] = copy.deepcopy(pvec.x)
f.close()
#sing_vals = np.loadtxt('singular_values.dat',dtype=np.int32)
sing_vals = np.loadtxt(sing_file, dtype=np.int32)
#sing_vals = [2]
print 'loading pest control file', pst_name
pst = phand.pst(pst_name)
print 'loading jco file', jco_name
jco = mhand.matrix()
jco.from_binary(jco_name)
print 'dropping zero-weight obs from jco'
drop_row_names = []
nz_weights, nz_names = [], []
for name, weight in zip(pst.observation_data.obsnme,
pst.observation_data.weight):
if weight == 0.0:
drop_row_names.append(name)
else:
nz_weights.append(weight)
nz_names.append(name)
jco.drop(drop_row_names)
print 'building obs cov matrix'
obs_unc = mhand.uncert(nz_names)
obs_unc.from_uncfile('simple_obs.unc')
print 'inverting obs cov matrix = q'
print 'and square root of q'
#q = np.linalg.inv(obs_unc.x)
#qhalf= linalg.sqrtm(q).real
u, s, vt = linalg.svd(obs_unc.x)
for i in range(u.shape[0]):
u[:, i] *= s[i]**(-0.5)
qhalf = np.dot(u, vt)
print 'build q^(1/2)X'
qX = np.dot(qhalf, jco.x)
if verbose:
np.savetxt('qX_py.mat', qX, fmt='%15.6E')
np.savetxt('qhalf_py.mat', qhalf, fmt='%15.6E')
np.savetxt('X_py.mat', jco.x, fmt='%15.6E')
jco_struct = None
spv_dict = None
if struct_file:
print 'loading structural parameter list'
spar_names, idxs = [], []
#f = open('struct_pars.dat','r')
f = open(struct_file, 'r')
for line in f:
name = line.strip()
if name in jco.col_names:
spar_names.append(name)
idxs.append(jco.col_names.index(name))
f.close()
print 'extracting omitted elements from q^(1/2)X'
jco_struct_q = qX[:, idxs]
jco_struct = jco.x[:, idxs]
qX = np.delete(qX, idxs, 1)
X = np.delete(jco.x, idxs, 1)
print 'forming omitted pred vectors'
spv_dict = {}
for out_name, pv in pv_dict.iteritems():
spv = np.atleast_2d(pv[idxs])
pv = np.atleast_2d(np.delete(pv, idxs)).transpose()
pv_dict[out_name] = pv
spv_dict[out_name] = spv
if 'halfscaled' in in_filename:
raise
print 'loading param uncert file for unscaled struct parameters'
unc = mhand.uncert(spar_names)
unc.from_uncfile('simple.unc')
#unc_struct = unc.x[idxs:idxs]
print unc_struct.shape
par_cov = None
struct_cov = None
if 'scaled' not in pst_name:
print 'building par cov matrix'
nspar_names = []
for name in jco.col_names:
if name not in spar_names:
nspar_names.append(name)
par_unc = mhand.uncert(nspar_names)
par_unc.from_uncfile('simple.unc')
if verbose:
np.savetxt('pycp.mat', par_unc.x, fmt='%15.6E')
print 'building structual par cov matrix'
struct_unc = mhand.uncert(spar_names)
struct_unc.from_uncfile('simple.unc')
par_cov = par_unc.x
struct_cov = struct_unc.x
pv = pvar.predvar(qX,pv_dict,sing_vals,verbose=verbose,jco_struct=jco_struct_q,\
pred_vectors_struct=spv_dict,struct_cov=struct_cov,par_cov=par_cov)
#pv = pvar.predvar(X,pv_dict,sing_vals,verbose=verbose,jco_struct=jco_struct,\
# pred_vectors_struct=spv_dict,struct_cov=struct_cov,par_cov=par_cov,obs_cov=obs_unc.x)
pv.calc()
start = 0
end = 10
parameters = []
pnames = []
for pnum in range(npar):
pname = 'p'+str(pnum+1)
pnames.append(pname)
par = parameter(pname,pnum+1,start,end,val)
parameters.append(par)
ins_file = os.path.join(tdir,"test.ins")
out_file = os.path.join(tdir,"test.bak.out")
tpl_file = os.path.join(tdir,"test.tpl")
write_template_file(tpl_file,parameters)
obs_names = write_insfile(ins_file,out_file,"obs_",nobs)
pst = phand.pst("test.bak.pst")
#"PARNME PARTRANS PARCHGLIM PARVAL1 PARLBND PARUBND PARGP SCALE OFFSET DERCOM"
pvals = np.ones(len(pnames))
partrans = ["log"] * len(pnames)
parchglim = ["factor"] * len(pnames)
parlbnd,parubnd = np.zeros(len(pnames))+0.1,np.ones(len(pnames))+2.0
pargp = ["par"] * len(pnames)
scale = np.ones(len(pnames))
offest = np.zeros(len(pnames))
dercom = np.ones(len(pnames))
par = pandas.DataFrame({"parnme":pnames,"partrans":partrans,"parchglim":parchglim,\
"parval1":pvals,"parlbnd":parlbnd,"parubnd":parubnd,"pargp":pargp,"scale":scale,\
"offset":offset,"dercom":dercom},index=pnames,columns=pst.par_fieldnames)
pst.parameter_data = par
print pst.parameter_data.columns
import pst_handler as phand
pst = phand.pst("reg.pst")
#pst.proportional_weights(0.3,0.5)
obs = pst.observation_data
obs.index = obs.obgnme
imas_grp = obs.groupby(lambda x: "imas" in x.lower()).groups
obs.weight.loc[imas_grp[False]] = 0.0
obs.index = obs.obsnme
big_groups = obs.groupby(
lambda x: x.startswith("ks062") or x.startswith("mb004")).groups
obs.weight.loc[big_groups[True]] = 0.0
obs = obs.sort(["obgnme", "obsval"])
pst.observation_data = obs
pst.zero_order_tikhonov()
pst.write("reg_imas.pst", True)
pst = phand.pst("reg.pst")
#pst.proportional_weights(0.25,1.0)
obs = pst.observation_data
obs.index = obs.obgnme
imas_grp = obs.groupby(lambda x: "imas" in x.lower()).groups
obs.weight.loc[imas_grp[True]] = 0.0
obs.index = obs.obsnme
big_groups = obs.groupby(
lambda x: x.startswith("ks062") or x.startswith("mb004")).groups
obs.weight.loc[big_groups[True]] = 0.0
obs = obs.sort(["obgnme", "obsval"])
pst.observation_data = obs
pst.zero_order_tikhonov()
pst.write("reg_gs.pst", True)
import numpy as np
import pandas
import pst_handler as ph
pst = ph.pst('umd03.pst')
blu_fix = ['petm02','petm07','petm08','petm09']
for pidx,par in pst.parameter_data.iterrows():
for fix in blu_fix:
if par['parnme'].startswith(fix):
pst.parameter_data.parval1[pidx] = 0.0
pst.parameter_data.partrans[pidx] = 'fixed'
pass
pst.reconcile_prior_2_pars()
pst.update()
pst.write_pst('umd03_blufixed.pst')
obs = pst.observation_data
obs.index = pandas.MultiIndex.from_arrays((obs.obsnme,obs.obgnme))
res = df = pandas.read_csv('umd03.res',sep='\s+',index_col=[0,1])
obs = obs.merge(res,left_index=True,right_index=True)
#--get the contribution of stage/head and baseflow to obj func
#--the baseflow group names end with 's'
resid_grouped = obs['Weight*Residual'].groupby(lambda x:x.endswith('s'),level=1)
phi_components = resid_grouped.aggregate(lambda x:np.sum(x**2))
print phi_components.values
#--mod bf weights they contribute a fraction of all the other obs to the obj function
bf_frac = 0.25
frac = np.sqrt((bf_frac*phi_components[False]) / phi_components[True])
import numpy as np
import pylab
import pandas
import pst_handler as ph
pst = ph.pst('umd02_processed_head.pst')
pst.observation_data.index = pst.observation_data.obsnme
df_file = '\\\\IGSBAMEWMS152\\Home\jwhite\MiamiDade\\umd02_la_master\\sliced\\i64predunc1_unscaled.csv'
df = pandas.read_csv(df_file,sep='|',index_col=0)
pred_obs = pst.observation_data.ix[df.index.values].dropna()
print pst.observation_data.obsval.shape,df.shape
df = np.abs(df.div(pred_obs.obsval,axis=0)* 100.0)
col_strings = df.columns
rawpro,bf,prepost = [],[],[]
for c in col_strings:
if 'raw' in c:
rawpro.append('raw')
else:
rawpro.append('processed')
if 'bf' in c:
bf.append('with_bf')
else:
bf.append('no_bf')
if 'pre' in c:
prepost.append('pre-cal')
else:
prepost.append('post-cal')
tups = zip(*(rawpro,bf,prepost))