def dataworth_test():
import os
import numpy as np
from pyemu import Schur
w_dir = os.path.join("..","..","verification","Freyberg")
forecasts = ["travel_time","sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
sc.pst.observation_data.index = sc.pst.observation_data.obsnme
base_obs = sc.pst.nnz_obs_names
zw_names = [name for name in sc.pst.zero_weight_obs_names if name not in forecasts ]
#print(sc.get_removed_obs_importance(obslist_dict={"test":zw_names}))
oname = "or00c00_0"
names = {"test":oname}
added = sc.get_added_obs_importance(base_obslist=base_obs,
obslist_dict=names,reset_zero_weight=True)
removed = sc.get_removed_obs_importance(obslist_dict=names)
print("removed",removed)
print("added",added)
for fname in forecasts:
diff = np.abs(removed.loc["test",fname] - added.loc["base",fname])
assert diff < 0.01,"{0},{1},{2}".format(fname,removed.loc["test",fname],added.loc["base",fname])
names = {"test1":oname,"test2":["or00c00_1","or00c00_2"]}
sc.next_most_important_added_obs(forecast="travel_time",obslist_dict=names,
base_obslist=sc.pst.nnz_obs_names)
def schur_test():
import os
from pyemu import Schur
w_dir = os.path.join("..","..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),forecasts=forecasts)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_contribution_dataframe_groups())
print(sc.get_importance_dataframe_groups())
def schur_test_nonpest():
import numpy as np
from pyemu import Matrix, Cov, Schur, Jco
#non-pest
pnames = ["p1", "p2", "p3"]
onames = ["o1", "o2", "o3", "o4"]
npar = len(pnames)
nobs = len(onames)
j_arr = np.random.random((nobs, npar))
jco = Jco(x=j_arr, row_names=onames, col_names=pnames)
parcov = Cov(x=np.eye(npar), names=pnames)
obscov = Cov(x=np.eye(nobs), names=onames)
forecasts = "o2"
s = Schur(jco=jco, parcov=parcov, obscov=obscov, forecasts=forecasts)
print(s.get_parameter_summary())
print(s.get_forecast_summary())
#this should fail
passed = False
try:
print(s.get_par_group_contribution())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
#this should fail
passed = False
try:
print(s.get_removed_obs_group_importance())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
print(s.get_par_contribution({"group1": ["p1", "p3"]}))
print(s.get_removed_obs_importance({"group1": ["o1", "o3"]}))
print(s.pandas)
forecasts = Matrix(x=np.random.random((1, npar)),
row_names=[forecasts],
col_names=pnames)
sc = Schur(jco=jco, forecasts=forecasts.T, parcov=parcov, obscov=obscov)
ffile = os.path.join("temp", "forecasts.jcb")
forecasts.to_binary(ffile)
sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov)
def css_test():
import os
import numpy as np
import pandas as pd
from pyemu import Schur
#w_dir = os.path.join("..","..","verification","10par_xsec","master_opt0")
w_dir = "la"
forecasts = ["h01_08", "h02_08"]
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"))
css = sc.get_par_css_dataframe()
css_pestpp = pd.read_csv(os.path.join(w_dir, "pest.isen"))
diff = (css_pestpp - css.pest_css).apply(np.abs).sum(axis=1)[0]
assert diff < 0.001, diff
def css_test():
import os
import numpy as np
import pandas as pd
from pyemu import Schur
#w_dir = os.path.join("..","..","verification","10par_xsec","master_opt0")
w_dir = "la"
forecasts = ["h01_08","h02_08"]
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"))
css = sc.get_par_css_dataframe()
css_pestpp = pd.read_csv(os.path.join(w_dir,"pest.isen"))
diff = (css_pestpp - css.pest_css).apply(np.abs).sum(axis=1)[0]
assert diff < 0.001,diff
def par_contrib_test():
import os
import numpy as np
from pyemu import Schur
w_dir = os.path.join("..","..","verification","Freyberg")
forecasts = ["travel_time","sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
par = sc.pst.parameter_data
par.index = par.parnme
groups = {name:list(idxs) for name,idxs in par.groupby(par.pargp).groups.items()}
parlist_dict = {}
print(sc.next_most_par_contribution(forecast="travel_time",
parlist_dict=groups))
def par_contrib_test():
import os
import numpy as np
from pyemu import Schur
w_dir = os.path.join("..","..","verification","Freyberg")
forecasts = ["travel_time","sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
par = sc.pst.parameter_data
par.index = par.parnme
groups = par.groupby(par.pargp).groups
parlist_dict = {}
print(sc.next_most_par_contribution(forecast="travel_time",
parlist_dict=groups))
def dataworth_test():
import os
import numpy as np
from pyemu import Schur,Cov
w_dir = os.path.join("..","verification","Freyberg")
forecasts = ["travel_time","sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
#sc.pst.observation_data.loc[sc.pst.nnz_obs_names[3:],"weight"] = 0.0
base_obs = sc.pst.nnz_obs_names
zw_names = [name for name in sc.pst.zero_weight_obs_names if name not in forecasts ]
#print(sc.get_removed_obs_importance(obslist_dict={"test":zw_names}))
oname = "or00c00_0"
names = {"test":oname}
added = sc.get_added_obs_importance(base_obslist=base_obs,
obslist_dict=names,reset_zero_weight=1.0)
sc.pst.observation_data.loc[oname,"weight"] = 1.0
#sc.reset_obscov()
cov = Cov.from_observation_data(sc.pst)
scm = Schur(jco=os.path.join(w_dir, "freyberg.jcb"), forecasts=forecasts, verbose=True,
obscov=cov)
scm.pst.observation_data.loc[oname, "weight"] = 1.0
removed = sc.get_removed_obs_importance(obslist_dict=names)
for fname in forecasts:
diff = np.abs(removed.loc["test",fname] - added.loc["base",fname])
print("add,{0},{1},{2}".format(fname, added.loc["test", fname], added.loc["base", fname]))
print("rem,{0},{1},{2}".format(fname, removed.loc["test", fname], removed.loc["base", fname]))
assert diff < 0.01,"{0},{1},{2}".format(fname,removed.loc["test",fname],added.loc["base",fname])
names = {"test1":oname,"test2":["or00c00_1","or00c00_2"]}
scm.pst.observation_data.loc[oname, "weight"] = 0.0
scm.next_most_important_added_obs(forecast="travel_time",obslist_dict=names,
base_obslist=scm.pst.nnz_obs_names,reset_zero_weight=1.0)
def la_test_io():
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_binary(os.path.join("temp", "pest.bin.cov"))
cov.to_ascii(os.path.join("temp", "pest.txt.cov"))
sc_bin = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp", "pest.bin.cov"))
sc_ascii = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=os.path.join("temp", "pest.txt.cov"))
def schur_test():
import os
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc"))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_removed_obs_group_importance())
def schur_test():
import os
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..","..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
pst = Pst(os.path.join(w_dir,"pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc"))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_removed_obs_group_importance())
def map_test():
import os
from pyemu import Schur
w_dir = os.path.join("..", "verification", "10par_xsec", "master_opt0")
forecasts = ["h01_08", "h02_08"]
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"), forecasts=forecasts)
print(sc.map_parameter_estimate)
print(sc.map_forecast_estimate)
def to_pearson_test():
import os
import numpy as np
import pandas as pd
from pyemu import Schur
#w_dir = os.path.join("..","..","verification","10par_xsec","master_opt0")
w_dir = "la"
forecasts = ["h01_08", "h02_08"]
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"))
sc.posterior_parameter.to_pearson()
def first_order_pearson_regul_test():
import os
from pyemu import Schur
from pyemu.utils.helpers import first_order_pearson_tikhonov, zero_order_tikhonov
w_dir = "la"
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"))
zero_order_tikhonov(sc.pst)
first_order_pearson_tikhonov(sc.pst, sc.posterior_parameter, reset=False)
print(sc.pst.prior_information)
sc.pst.rectify_pi()
def dataworth_next_test():
import os
import numpy as np
from pyemu import Schur
w_dir = os.path.join("..", "verification", "Freyberg")
#w_dir = os.path.join("..","..","examples","freyberg")
forecasts = ["sw_gw_0", "sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir, "freyberg.jcb"),
forecasts=forecasts,
verbose=True)
next_test = sc.next_most_important_added_obs(
forecast="sw_gw_0",
base_obslist=sc.pst.nnz_obs_names,
obslist_dict={"test": sc.pst.nnz_obs_names})
# the returned dataframe should only have one row since the 'base' case
# should be the same as the 'test' case
assert next_test.shape[0] == 1
obs = sc.pst.observation_data
obs.index = obs.obsnme
row_groups = obs.groupby([
lambda x: x.startswith("or"), lambda x: x in sc.pst.nnz_obs_names,
lambda x: x[:4]
]).groups
obslist_dict = {}
for key, idxs in row_groups.items():
if not key[0] or key[1]:
continue
obslist_dict[key[2]] = list(idxs)
imp_df = sc.get_added_obs_importance(base_obslist=sc.pst.nnz_obs_names,
obslist_dict=obslist_dict,
reset_zero_weight=1.0)
next_test = sc.next_most_important_added_obs(
forecast="sw_gw_0",
base_obslist=sc.pst.nnz_obs_names,
obslist_dict=obslist_dict,
reset_zero_weight=1.0,
niter=4)
print(next_test)
print(imp_df.sort_index())
assert next_test.shape[0] == 4
def pearson_regul_test():
import os
import numpy as np
import pandas as pd
from pyemu import Schur
from pyemu.utils.helpers import first_order_pearson_tikhonov
#w_dir = os.path.join("..","..","verification","10par_xsec","master_opt0")
w_dir = "la"
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"))
sc.pst.zero_order_tikhonov()
first_order_pearson_tikhonov(sc.pst, sc.posterior_parameter, reset=False)
print(sc.pst.prior_information)
def first_order_pearson_regul_test():
import os
from pyemu import Schur
from pyemu.utils.helpers import first_order_pearson_tikhonov, zero_order_tikhonov
w_dir = "la"
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"))
pt = sc.posterior_parameter
zero_order_tikhonov(sc.pst)
first_order_pearson_tikhonov(sc.pst, pt, reset=False)
print(sc.pst.prior_information)
sc.pst.rectify_pi()
assert sc.pst.control_data.pestmode == "regularization"
sc.pst.write(os.path.join('temp', 'test.pst'))
def schur_test_nonpest():
import numpy as np
from pyemu import Matrix, Cov, Schur, Jco
#non-pest
pnames = ["p1","p2","p3"]
onames = ["o1","o2","o3","o4"]
npar = len(pnames)
nobs = len(onames)
j_arr = np.random.random((nobs,npar))
jco = Jco(x=j_arr,row_names=onames,col_names=pnames)
parcov = Cov(x=np.eye(npar),names=pnames)
obscov = Cov(x=np.eye(nobs),names=onames)
forecasts = "o2"
s = Schur(jco=jco,parcov=parcov,obscov=obscov,forecasts=forecasts)
print(s.get_parameter_summary())
print(s.get_forecast_summary())
#this should fail
passed = False
try:
print(s.get_par_group_contribution())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
#this should fail
passed = False
try:
print(s.get_removed_obs_group_importance())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
print(s.get_par_contribution({"group1":["p1","p3"]}))
print(s.get_removed_obs_importance({"group1":["o1","o3"]}))
print(s.pandas)
forecasts = Matrix(x=np.random.random((1,npar)),row_names=[forecasts],col_names=pnames)
sc = Schur(jco=jco,forecasts=forecasts.T,parcov=parcov,obscov=obscov)
ffile = os.path.join("temp","forecasts.jcb")
forecasts.to_binary(ffile)
sc = Schur(jco=jco, forecasts=ffile, parcov=parcov, obscov=obscov)
def dataworth_next_test():
import os
import numpy as np
from pyemu import Schur
w_dir = os.path.join("..","..","verification","Freyberg")
#w_dir = os.path.join("..","..","examples","freyberg")
forecasts = ["sw_gw_0","sw_gw_1"]
sc = Schur(jco=os.path.join(w_dir,"freyberg.jcb"),forecasts=forecasts,verbose=True)
next_test = sc.next_most_important_added_obs(forecast="sw_gw_0",
base_obslist=sc.pst.nnz_obs_names,
obslist_dict={"test":sc.pst.nnz_obs_names})
# the returned dataframe should only have one row since the 'base' case
# should be the same as the 'test' case
assert next_test.shape[0] == 1
obs = sc.pst.observation_data
obs.index = obs.obsnme
row_groups = obs.groupby([lambda x: x.startswith("or"), lambda x: x in sc.pst.nnz_obs_names,
lambda x: x[:4]]).groups
obslist_dict = {}
for key,idxs in row_groups.items():
if not key[0] or key[1]:
continue
obslist_dict[key[2]] = idxs
imp_df = sc.get_added_obs_importance(base_obslist=sc.pst.nnz_obs_names,
obslist_dict=obslist_dict,
reset_zero_weight=1.0)
next_test = sc.next_most_important_added_obs(forecast="sw_gw_0",
base_obslist=sc.pst.nnz_obs_names,
obslist_dict=obslist_dict,
reset_zero_weight=1.0,
niter=4)
print(next_test)
print(imp_df.sort_index())
assert next_test.shape[0] == 4
def schur_test():
import os
import numpy as np
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..","verification","henry")
forecasts = ["pd_ten","c_obs10_2"]
pst = Pst(os.path.join(w_dir,"pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp","pest.unc"),covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp","pest.unc"))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
print(sc.get_parameter_summary(include_map=True))
print(sc.get_forecast_summary(include_map=True))
print(sc.get_removed_obs_importance(reset_zero_weight=True))
sc = Schur(jco=os.path.join(w_dir,"pest.jcb"),
forecasts=forecasts,
sigma_range=6.0)
cov = Cov.from_parameter_data(pst,sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0,scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
pst.parameter_data.loc[:,"offset"] = 100.0
cov = Cov.from_parameter_data(pst)
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
pst=pst,
forecasts=forecasts,
sigma_range=6.0, scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0
cov = Cov.from_parameter_data(pst,scale_offset=False,sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def schur_test_nonpest():
import numpy as np
from pyemu import Matrix, Cov, Schur
#non-pest
pnames = ["p1","p2","p3"]
onames = ["o1","o2","o3","o4"]
npar = len(pnames)
nobs = len(onames)
j_arr = np.random.random((nobs,npar))
jco = Matrix(x=j_arr,row_names=onames,col_names=pnames)
parcov = Cov(x=np.eye(npar),names=pnames)
obscov = Cov(x=np.eye(nobs),names=onames)
forecasts = "o2"
s = Schur(jco=jco,parcov=parcov,obscov=obscov,forecasts=forecasts)
print(s.get_parameter_summary())
print(s.get_forecast_summary())
#this should fail
passed = False
try:
print(s.get_contribution_dataframe_groups())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
#this should fail
passed = False
try:
print(s.get_importance_dataframe_groups())
passed = True
except Exception as e:
print(str(e))
if passed:
raise Exception("should have failed")
print(s.get_contribution_dataframe({"group1":["p1","p3"]}))
print(s.get_importance_dataframe({"group1":["o1","o3"]}))
def to_mps(jco,
obj_func=None,
obs_constraint_sense=None,
pst=None,
decision_var_names=None,
mps_filename=None,
risk=0.5):
"""helper utility to write an mps file from pest-style
jacobian matrix. Requires corresponding pest control
file.
Parameters:
jco : pyemu.Matrix or str (filename of matrix)
obj_func : optional. If None, an obs group must exist
named 'n' and must have one one member. Can be a str, which
is the name of an observation to treat as the objective function
or can be a dict, which is keyed on decision var names and valued
with objective function coeffs.
obs_constraint_sense : optional. If None, obs groups are sought that
have names "l","g", or "e" - members of these groups are treated
as constraints. Otherwise, must be a dict keyed on constraint
(obs) names with values of "l","g", or "e".
pst : optional. If None, a pest control file is sought with
filename <case>.pst. Otherwise, must be a pyemu.Pst instance or
a filename of a pest control file. The control must have an
associated .res or .rei file - this is needed for the RHS of the
constraints.
decision_var_names: optional. If None, all parameters are treated as
decision vars. Otherwise, must be a list of str of parameter names
to use as decision vars
mps_filename : optional. If None, then <case>.mps is written.
Otherwise, must be a str.
risk : float
the level of risk tolerance/aversion in the chance constraints.
Values other then 0.50 require at least one parameter (non decision
var) in the jco. Ranges from 0.0,1.0
"""
#if jco arg is a string, load a jco from binary
if isinstance(jco, str):
pst_name = jco.lower().replace('.jcb', ".pst").replace(".jco", ".pst")
jco = Matrix.from_binary(jco)
assert isinstance(jco, Matrix)
# try to find a pst
if pst is None:
if os.path.exists(pst_name):
pst = Pst(pst_name)
else:
raise Exception("could not find pst file {0} and pst argument is None, a ".format(pst_name) +\
"pst instance is required for setting decision variable bound constraints")
else:
assert len(set(jco.row_names).difference(
pst.observation_data.index)) == 0
assert len(set(jco.col_names).difference(
pst.parameter_data.index)) == 0
#make sure the pst has an associate res
assert pst.res is not None," could find a residuals file (.res or .rei) for" +\
" for control file {0}".format(pst.filename)
#if no decision_var_names where passed, use all columns in the jco
if decision_var_names is None:
decision_var_names = jco.col_names
#otherwise, do some error checking and processing
else:
if not isinstance(decision_var_names, list):
decision_var_names = [decision_var_names]
for i, dv in enumerate(decision_var_names):
dv = dv.lower()
decision_var_names[i] = dv
assert dv in jco.col_names, "decision var {0} not in jco column names".format(
dv)
assert dv in pst.parameter_data.index, "decision var {0} not in pst parameter names".format(
dv)
#if no obs_constraint_sense, try to build one from the obs group info
if obs_constraint_sense is None:
const_groups = [
grp for grp in pst.obs_groups if grp.lower() in OPERATOR_WORDS
]
if len(const_groups) == 0:
raise Exception("to_mps(): obs_constraint_sense is None and no "+\
"obseravtion groups in {0}".format(','.join(pst.obs_groups)))
obs_constraint_sense = {}
obs_groups = pst.observation_data.groupby(
pst.observation_data.obgnme).groups
for og, obs_names in obs_groups.items():
if og == 'n':
continue
if og in const_groups:
for oname in obs_names:
obs_constraint_sense[oname] = og
assert isinstance(obs_constraint_sense, dict)
assert len(obs_constraint_sense) > 0, "no obs_constraints..."
#build up a dict of (in)equality operators for the constraints
operators = {}
for obs_name, operator in obs_constraint_sense.items():
obs_name = obs_name.lower()
assert obs_name in pst.obs_names, "obs constraint {0} not in pst observation names"
assert obs_name in pst.res.name, " obs constraint {0} not in pst.res names"
assert obs_name in jco.row_names, "obs constraint {0} not in jco row names".format(
obs_name)
if operator.lower() not in OPERATOR_WORDS:
if operator not in OPERATOR_SYMBOLS:
raise Exception("operator {0} not in [{1}] or [{2}]".\
format(operator,','.join(OPERATOR_WORDS),','\
.join(OPERATOR_SYMBOLS)))
op = OPERATOR_WORDS[OPERATOR_SYMBOLS.index(operator)]
else:
op = operator.lower()
operators[obs_name] = op
obs_constraint_sense[obs_name.lower()] = obs_constraint_sense.\
pop(obs_name)
#build a list of constaint names in order WRT jco row order
# order_obs_constraints = [name for name in jco.row_names if name in
# obs_constraint_sense]
order_obs_constraints = list(obs_constraint_sense.keys())
order_obs_constraints.sort()
#build a list of decision var names in order WRT jco col order
#order_dec_var = [name for name in jco.col_names if name in
# decision_var_names]
order_dec_var = list(decision_var_names)
order_dec_var.sort()
#shorten constraint names if needed
new_const_count = 0
new_constraint_names = {}
for name in order_obs_constraints:
if len(name) > 8:
new_name = name[:7] + "{0}".format(new_const_count)
print("to_mps(): shortening constraint name {0} to {1}\n".format(
name, new_name))
new_constraint_names[name] = new_name
new_const_count += 1
else:
new_constraint_names[name] = name
#shorten decision var names if needed
new_dec_count = 0
new_decision_names = {}
for name in order_dec_var:
if len(name) > 8:
new_name = name[:7] + "{0}".format(new_dec_count)
print("to_mps(): shortening decision var name {0} to {1}\n".format(
name, new_name))
new_decision_names[name] = new_name
new_dec_count += 1
else:
new_decision_names[name] = name
# if no obj_func, try to make one
if obj_func is None:
# look for an obs group named 'n' with a single member
og = pst.obs_groups
if 'n' not in pst.obs_groups:
raise Exception("to_mps(): obj_func is None but no "+\
"obs group named 'n'")
grps = pst.observation_data.groupby(pst.observation_data.obgnme).groups
assert len(grps["n"]) == 1,"to_mps(): 'n' obj_func group has more " +\
" than one member, mps only support one objf "
obj_name = grps['n'][0]
obj_iidx = jco.row_names.index(obj_name)
obj = {}
for name in order_dec_var:
jco_jidx = jco.col_names.index(name)
obj[name] = jco.x[obj_iidx, jco_jidx]
#otherwise, parse what was passed
elif isinstance(obj_func, str):
obj_func = obj_func.lower()
assert obj_func in jco.row_names,\
"obj_func {0} not in jco.row_names".format(obj_func)
assert obj_func in pst.observation_data.obsnme,\
"obj_func {0} not in pst observations".format(obj_func)
obj_iidx = jco.row_names.index(obj_func)
obj = {}
for name in order_dec_var:
jco_jidx = jco.col_names.index(name)
obj[name] = jco.x[obj_iidx, jco_jidx]
obj_name = str(obj_func)
elif isinstance(obj_func, dict):
obj = {}
for name, value in obj_func.items():
assert name in jco.col_names,"to_mps(): obj_func key "+\
"{0} not ".format(name) +\
"in jco col names"
obj[name] = float(value)
obj_name = "obj_func"
else:
raise NotImplementedError("unsupported obj_func arg type {0}".format(\
type(obj_func)))
if risk != 0.5:
try:
from scipy.special import erfinv
except Exception as e:
raise Exception("to_mps() error importing erfinv from scipy.special: "+\
"{0}".format(str(e)))
par_names = [
name for name in jco.col_names if name not in decision_var_names
]
if len(par_names) == 0:
raise Exception("to_mps() error: risk != 0.5, but no "+\
"non-decision vars parameters ")
unc_jco = jco.get(col_names=par_names)
unc_pst = pst.get(par_names=par_names)
sc = Schur(jco=unc_jco, pst=unc_pst, forecasts=order_obs_constraints)
constraint_std = sc.get_forecast_summary().loc[:, "post_var"].apply(
np.sqrt)
rhs = {}
# the probit value for a given risk...using the inverse
# error function
probit_val = np.sqrt(2.0) * erfinv((2.0 * risk) - 1.0)
for name in order_obs_constraints:
mu = unc_pst.res.loc[name, "residual"]
std = constraint_std.loc[name]
#if this is a less than constraint, then we want
# to subtract
if operators[name] == 'l':
prob_val = mu - (probit_val * std)
#if this is a greater than constraint, then we want
# to add
elif operators[name] == "g":
prob_val = mu + (probit_val * std)
else:
raise NotImplementedError("chance constraints only " +\
"implemented for 'l' or 'g' " +\
"type constraints, not " +\
"{0}".format(operators[name]))
rhs[name] = prob_val
else:
rhs = {n: pst.res.loc[n, "residual"] for n in order_obs_constraints}
if mps_filename is None:
mps_filename = pst.filename.replace(".pst", ".mps")
with open(mps_filename, 'w') as f:
f.write("NAME {0}\n".format("pest_opt"))
f.write("ROWS\n")
for name in order_obs_constraints:
f.write(" {0} {1}\n".format(operators[name],
new_constraint_names[name]))
f.write(" {0} {1}\n".format('n', obj_name))
f.write("COLUMNS\n")
for dname in order_dec_var:
jco_jidx = jco.col_names.index(dname)
for cname in order_obs_constraints:
jco_iidx = jco.row_names.index(cname)
v = jco.x[jco_iidx, jco_jidx]
f.write(" {0:8} {1:8} {2:10G}\n".\
format(new_decision_names[dname],
new_constraint_names[cname],
v))
# f.write(" {0:8} {1:8} {2:10G}\n".\
# format(new_decision_names[dname],
# obj_name,pst.parameter_data.loc[dname,"parval1"]))
f.write(" {0:8} {1:8} {2:10G}\n".\
format(new_decision_names[dname],
obj_name,obj_func[dname]))
f.write("RHS\n")
for iname, name in enumerate(order_obs_constraints):
f.write(" {0:8} {1:8} {2:10G}\n".format(
"rhs", new_constraint_names[name], rhs[name]))
f.write("BOUNDS\n")
for name in order_dec_var:
up,lw = pst.parameter_data.loc[name,"parubnd"],\
pst.parameter_data.loc[name,"parlbnd"]
f.write(" {0:2} {1:8} {2:8} {3:10G}\n".\
format("UP","BOUND",name,up))
f.write(" {0:2} {1:8} {2:8} {3:10G}\n".\
format("LO","BOUND",name,lw))
f.write("ENDATA\n")
def schur_test():
import os
import numpy as np
from pyemu import Schur, Cov, Pst
w_dir = os.path.join("..", "verification", "henry")
forecasts = ["pd_ten", "c_obs10_2"]
pst = Pst(os.path.join(w_dir, "pest.pst"))
cov = Cov.from_parameter_data(pst)
cov.to_uncfile(os.path.join("temp", "pest.unc"), covmat_file=None)
cov2 = Cov.from_uncfile(os.path.join("temp", "pest.unc"))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
parcov=cov2)
print(sc.prior_forecast)
print(sc.posterior_forecast)
print(sc.get_par_group_contribution())
df = sc.get_par_group_contribution(include_prior_results=True)
levels = list(df.columns.levels[1])
assert "prior" in levels, levels
assert "post" in levels, levels
print(sc.get_parameter_summary(include_map=True))
print(sc.get_forecast_summary(include_map=True))
print(sc.get_removed_obs_importance(reset_zero_weight=True))
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0)
cov = Cov.from_parameter_data(pst, sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
forecasts=forecasts,
sigma_range=6.0,
scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
pst.parameter_data.loc[:, "offset"] = 100.0
cov = Cov.from_parameter_data(pst)
sc = Schur(jco=os.path.join(w_dir, "pest.jcb"),
pst=pst,
forecasts=forecasts,
sigma_range=6.0,
scale_offset=False)
assert np.abs((sc.parcov.x - cov.x).sum()) != 0.0
cov = Cov.from_parameter_data(pst, scale_offset=False, sigma_range=6.0)
assert np.abs((sc.parcov.x - cov.x).sum()) == 0.0
def to_mps(jco,obj_func=None,obs_constraint_sense=None,pst=None,
decision_var_names=None,mps_filename=None,
risk=0.5):
"""helper utility to write an mps file from pest-style
jacobian matrix. Requires corresponding pest control
file.
Parameters:
jco : pyemu.Matrix or str (filename of matrix)
obj_func : optional. If None, an obs group must exist
named 'n' and must have one one member. Can be a str, which
is the name of an observation to treat as the objective function
or can be a dict, which is keyed on decision var names and valued
with objective function coeffs.
obs_constraint_sense : optional. If None, obs groups are sought that
have names "l","g", or "e" - members of these groups are treated
as constraints. Otherwise, must be a dict keyed on constraint
(obs) names with values of "l","g", or "e".
pst : optional. If None, a pest control file is sought with
filename <case>.pst. Otherwise, must be a pyemu.Pst instance or
a filename of a pest control file. The control must have an
associated .res or .rei file - this is needed for the RHS of the
constraints.
decision_var_names: optional. If None, all parameters are treated as
decision vars. Otherwise, must be a list of str of parameter names
to use as decision vars
mps_filename : optional. If None, then <case>.mps is written.
Otherwise, must be a str.
risk : float
the level of risk tolerance/aversion in the chance constraints.
Values other then 0.50 require at least one parameter (non decision
var) in the jco. Ranges from 0.0,1.0
"""
#if jco arg is a string, load a jco from binary
if isinstance(jco,str):
pst_name = jco.lower().replace('.jcb',".pst").replace(".jco",".pst")
jco = Matrix.from_binary(jco)
assert isinstance(jco,Matrix)
# try to find a pst
if pst is None:
if os.path.exists(pst_name):
pst = Pst(pst_name)
else:
raise Exception("could not find pst file {0} and pst argument is None, a ".format(pst_name) +\
"pst instance is required for setting decision variable bound constraints")
else:
assert len(set(jco.row_names).difference(pst.observation_data.index)) == 0
assert len(set(jco.col_names).difference(pst.parameter_data.index)) == 0
#make sure the pst has an associate res
assert pst.res is not None," could find a residuals file (.res or .rei) for" +\
" for control file {0}".format(pst.filename)
#if no decision_var_names where passed, use all columns in the jco
if decision_var_names is None:
decision_var_names = jco.col_names
#otherwise, do some error checking and processing
else:
if not isinstance(decision_var_names,list):
decision_var_names = [decision_var_names]
for i,dv in enumerate(decision_var_names):
dv = dv.lower()
decision_var_names[i] = dv
assert dv in jco.col_names,"decision var {0} not in jco column names".format(dv)
assert dv in pst.parameter_data.index,"decision var {0} not in pst parameter names".format(dv)
#if no obs_constraint_sense, try to build one from the obs group info
if obs_constraint_sense is None:
const_groups = [grp for grp in pst.obs_groups if grp.lower() in OPERATOR_WORDS]
if len(const_groups) == 0:
raise Exception("to_mps(): obs_constraint_sense is None and no "+\
"obseravtion groups in {0}".format(','.join(pst.obs_groups)))
obs_constraint_sense = {}
obs_groups = pst.observation_data.groupby(pst.observation_data.obgnme).groups
for og,obs_names in obs_groups.items():
if og == 'n':
continue
if og in const_groups:
for oname in obs_names:
obs_constraint_sense[oname] = og
assert isinstance(obs_constraint_sense,dict)
assert len(obs_constraint_sense) > 0,"no obs_constraints..."
#build up a dict of (in)equality operators for the constraints
operators = {}
for obs_name,operator in obs_constraint_sense.items():
obs_name = obs_name.lower()
assert obs_name in pst.obs_names,"obs constraint {0} not in pst observation names"
assert obs_name in pst.res.name," obs constraint {0} not in pst.res names"
assert obs_name in jco.row_names,"obs constraint {0} not in jco row names".format(obs_name)
if operator.lower() not in OPERATOR_WORDS:
if operator not in OPERATOR_SYMBOLS:
raise Exception("operator {0} not in [{1}] or [{2}]".\
format(operator,','.join(OPERATOR_WORDS),','\
.join(OPERATOR_SYMBOLS)))
op = OPERATOR_WORDS[OPERATOR_SYMBOLS.index(operator)]
else:
op = operator.lower()
operators[obs_name] = op
obs_constraint_sense[obs_name.lower()] = obs_constraint_sense.\
pop(obs_name)
#build a list of constaint names in order WRT jco row order
# order_obs_constraints = [name for name in jco.row_names if name in
# obs_constraint_sense]
order_obs_constraints = list(obs_constraint_sense.keys())
order_obs_constraints.sort()
#build a list of decision var names in order WRT jco col order
#order_dec_var = [name for name in jco.col_names if name in
# decision_var_names]
order_dec_var = list(decision_var_names)
order_dec_var.sort()
#shorten constraint names if needed
new_const_count = 0
new_constraint_names = {}
for name in order_obs_constraints:
if len(name) > 8:
new_name = name[:7]+"{0}".format(new_const_count)
print("to_mps(): shortening constraint name {0} to {1}\n".format(name,new_name))
new_constraint_names[name] = new_name
new_const_count += 1
else:
new_constraint_names[name] = name
#shorten decision var names if needed
new_dec_count = 0
new_decision_names = {}
for name in order_dec_var:
if len(name) > 8:
new_name = name[:7]+"{0}".format(new_dec_count)
print("to_mps(): shortening decision var name {0} to {1}\n".format(name,new_name))
new_decision_names[name] = new_name
new_dec_count += 1
else:
new_decision_names[name] = name
# if no obj_func, try to make one
if obj_func is None:
# look for an obs group named 'n' with a single member
og = pst.obs_groups
if 'n' not in pst.obs_groups:
raise Exception("to_mps(): obj_func is None but no "+\
"obs group named 'n'")
grps = pst.observation_data.groupby(pst.observation_data.obgnme).groups
assert len(grps["n"]) == 1,"to_mps(): 'n' obj_func group has more " +\
" than one member, mps only support one objf "
obj_name = grps['n'][0]
obj_iidx = jco.row_names.index(obj_name)
obj = {}
for name in order_dec_var:
jco_jidx = jco.col_names.index(name)
obj[name] = jco.x[obj_iidx,jco_jidx]
#otherwise, parse what was passed
elif isinstance(obj_func,str):
obj_func = obj_func.lower()
assert obj_func in jco.row_names,\
"obj_func {0} not in jco.row_names".format(obj_func)
assert obj_func in pst.observation_data.obsnme,\
"obj_func {0} not in pst observations".format(obj_func)
obj_iidx = jco.row_names.index(obj_func)
obj = {}
for name in order_dec_var:
jco_jidx = jco.col_names.index(name)
obj[name] = jco.x[obj_iidx,jco_jidx]
obj_name = str(obj_func)
elif isinstance(obj_func,dict):
obj = {}
for name,value in obj_func.items():
assert name in jco.col_names,"to_mps(): obj_func key "+\
"{0} not ".format(name) +\
"in jco col names"
obj[name] = float(value)
obj_name = "obj_func"
else:
raise NotImplementedError("unsupported obj_func arg type {0}".format(\
type(obj_func)))
if risk != 0.5:
try:
from scipy.special import erfinv
except Exception as e:
raise Exception("to_mps() error importing erfinv from scipy.special: "+\
"{0}".format(str(e)))
par_names = [name for name in jco.col_names if name not in decision_var_names]
if len(par_names) == 0:
raise Exception("to_mps() error: risk != 0.5, but no "+\
"non-decision vars parameters ")
unc_jco = jco.get(col_names=par_names)
unc_pst = pst.get(par_names=par_names)
sc = Schur(jco=unc_jco,pst=unc_pst,forecasts=order_obs_constraints)
constraint_std = sc.get_forecast_summary().loc[:,"post_var"].apply(np.sqrt)
rhs = {}
# the probit value for a given risk...using the inverse
# error function
probit_val = np.sqrt(2.0) * erfinv((2.0 * risk) - 1.0)
for name in order_obs_constraints:
mu = unc_pst.res.loc[name,"residual"]
std = constraint_std.loc[name]
#if this is a less than constraint, then we want
# to subtract
if operators[name] == 'l':
prob_val = mu - (probit_val * std)
#if this is a greater than constraint, then we want
# to add
elif operators[name] == "g":
prob_val = mu + (probit_val * std)
else:
raise NotImplementedError("chance constraints only " +\
"implemented for 'l' or 'g' " +\
"type constraints, not " +\
"{0}".format(operators[name]))
rhs[name] = prob_val
else:
rhs = {n:pst.res.loc[n,"residual"] for n in order_obs_constraints}
if mps_filename is None:
mps_filename = pst.filename.replace(".pst",".mps")
with open(mps_filename,'w') as f:
f.write("NAME {0}\n".format("pest_opt"))
f.write("ROWS\n")
for name in order_obs_constraints:
f.write(" {0} {1}\n".format(operators[name],
new_constraint_names[name]))
f.write(" {0} {1}\n".format('n',obj_name))
f.write("COLUMNS\n")
for dname in order_dec_var:
jco_jidx = jco.col_names.index(dname)
for cname in order_obs_constraints:
jco_iidx = jco.row_names.index(cname)
v = jco.x[jco_iidx,jco_jidx]
f.write(" {0:8} {1:8} {2:10G}\n".\
format(new_decision_names[dname],
new_constraint_names[cname],
v))
# f.write(" {0:8} {1:8} {2:10G}\n".\
# format(new_decision_names[dname],
# obj_name,pst.parameter_data.loc[dname,"parval1"]))
f.write(" {0:8} {1:8} {2:10G}\n".\
format(new_decision_names[dname],
obj_name,obj_func[dname]))
f.write("RHS\n")
for iname,name in enumerate(order_obs_constraints):
f.write(" {0:8} {1:8} {2:10G}\n".
format("rhs",new_constraint_names[name],
rhs[name]))
f.write("BOUNDS\n")
for name in order_dec_var:
up,lw = pst.parameter_data.loc[name,"parubnd"],\
pst.parameter_data.loc[name,"parlbnd"]
f.write(" {0:2} {1:8} {2:8} {3:10G}\n".\
format("UP","BOUND",name,up))
f.write(" {0:2} {1:8} {2:8} {3:10G}\n".\
format("LO","BOUND",name,lw))
f.write("ENDATA\n")
