def setUp(self):
from pyomo.contrib.parmest.examples.reactor_design.reactor_design import reactor_design_model
# Data from the design
data = pd.DataFrame(data=[[1.05, 10000, 3458.4, 1060.8, 1683.9, 1898.5],
[1.10, 10000, 3535.1, 1064.8, 1613.3, 1893.4],
[1.15, 10000, 3609.1, 1067.8, 1547.5, 1887.8],
[1.20, 10000, 3680.7, 1070.0, 1486.1, 1881.6],
[1.25, 10000, 3750.0, 1071.4, 1428.6, 1875.0],
[1.30, 10000, 3817.1, 1072.2, 1374.6, 1868.0],
[1.35, 10000, 3882.2, 1072.4, 1324.0, 1860.7],
[1.40, 10000, 3945.4, 1072.1, 1276.3, 1853.1],
[1.45, 10000, 4006.7, 1071.3, 1231.4, 1845.3],
[1.50, 10000, 4066.4, 1070.1, 1189.0, 1837.3],
[1.55, 10000, 4124.4, 1068.5, 1148.9, 1829.1],
[1.60, 10000, 4180.9, 1066.5, 1111.0, 1820.8],
[1.65, 10000, 4235.9, 1064.3, 1075.0, 1812.4],
[1.70, 10000, 4289.5, 1061.8, 1040.9, 1803.9],
[1.75, 10000, 4341.8, 1059.0, 1008.5, 1795.3],
[1.80, 10000, 4392.8, 1056.0, 977.7, 1786.7],
[1.85, 10000, 4442.6, 1052.8, 948.4, 1778.1],
[1.90, 10000, 4491.3, 1049.4, 920.5, 1769.4],
[1.95, 10000, 4538.8, 1045.8, 893.9, 1760.8]],
columns=['sv', 'caf', 'ca', 'cb', 'cc', 'cd'])
theta_names = ['k1', 'k2', 'k3']
def SSE(model, data):
expr = (float(data['ca']) - model.ca)**2 + \
(float(data['cb']) - model.cb)**2 + \
(float(data['cc']) - model.cc)**2 + \
(float(data['cd']) - model.cd)**2
return expr
self.pest = parmest.Estimator(reactor_design_model, data, theta_names, SSE)
def test_covariance(self):
from pyomo.contrib.interior_point.inverse_reduced_hessian import inv_reduced_hessian_barrier
# Number of datapoints.
# 3 data components (ca, cb, cc), 20 timesteps, 1 scenario = 60
# In this example, this is the number of data points in data_df, but that's
# only because the data is indexed by time and contains no additional inforamtion.
n = 60
# Compute covariance using parmest
obj, theta, cov = self.pest_df.theta_est(calc_cov=True, cov_n=n)
# Compute covariance using interior_point
vars_list = [self.m_df.k1, self.m_df.k2]
solve_result, inv_red_hes = inv_reduced_hessian_barrier(
self.m_df, independent_variables=vars_list, tee=True)
l = len(vars_list)
cov_interior_point = 2 * obj / (n - l) * inv_red_hes
cov_interior_point = pd.DataFrame(cov_interior_point, ['k1', 'k2'],
['k1', 'k2'])
cov_diff = (cov - cov_interior_point).abs().sum().sum()
self.assertTrue(cov.loc['k1', 'k1'] > 0)
self.assertTrue(cov.loc['k2', 'k2'] > 0)
self.assertAlmostEqual(cov_diff, 0, places=6)
def setUp(self):
from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import rooney_biegler_model
# Note, the data used in this test has been corrected to use data.loc[5,'hour'] = 7 (instead of 6)
data = pd.DataFrame(data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0],
[5, 15.6], [7, 19.8]],
columns=['hour', 'y'])
theta_names = ['asymptote', 'rate_constant']
def SSE(model, data):
expr = sum((data.y[i] - model.response_function[data.hour[i]])**2
for i in data.index)
return expr
solver_options = {
'tol': 1e-8,
}
self.data = data
self.pest = parmest.Estimator(rooney_biegler_model,
data,
theta_names,
SSE,
solver_options=solver_options)
def theta_est_leaveNout(self, lNo, lNo_samples=None, seed=None,
return_samples=False):
"""
Parameter estimation where N data points are left out of each sample
Parameters
----------
lNo: int
Number of data points to leave out for parameter estimation
lNo_samples: int
Number of leave-N-out samples. If lNo_samples=None, the maximum
number of combinations will be used
seed: int or None, optional
Random seed
return_samples: bool, optional
Return a list of sample numbers that were left out
Returns
-------
lNo_theta: DataFrame
Theta values for each sample and (if return_samples = True)
the sample numbers left out of each estimation
"""
assert isinstance(lNo, int)
assert isinstance(lNo_samples, (type(None), int))
assert isinstance(seed, (type(None), int))
assert isinstance(return_samples, bool)
samplesize = len(self._numbers_list)-lNo
if seed is not None:
np.random.seed(seed)
global_list = self._get_sample_list(samplesize, lNo_samples, replacement=False)
task_mgr = mpiu.ParallelTaskManager(len(global_list))
local_list = task_mgr.global_to_local_data(global_list)
# Reset numbers_list
self._numbers_list = list(range(samplesize))
lNo_theta = list()
for idx, sample in local_list:
objval, thetavals = self.theta_est(bootlist=list(sample))
lNo_s = list(set(range(len(self.callback_data))) - set(sample))
thetavals['lNo'] = np.sort(lNo_s)
lNo_theta.append(thetavals)
# Reset numbers_list (back to original)
self._numbers_list = list(range(len(self.callback_data)))
global_bootstrap_theta = task_mgr.allgather_global_data(lNo_theta)
lNo_theta = pd.DataFrame(global_bootstrap_theta)
if not return_samples:
del lNo_theta['lNo']
return lNo_theta
def test_initializer_initializer(self):
d = {'col1': [1, 2, 4], 'col2': [10, 20, 40]}
df = pd.DataFrame(data=d)
a = Initializer(DataFrameInitializer(df, 'col2'))
self.assertIs(type(a), DataFrameInitializer)
self.assertFalse(a.constant())
self.assertFalse(a.verified)
self.assertTrue(a.contains_indices())
self.assertEqual(list(a.indices()), [0, 1, 2])
self.assertEqual(a(None, 0), 10)
self.assertEqual(a(None, 1), 20)
self.assertEqual(a(None, 2), 40)
def test_dataframe(self):
d = {'col1': [1, 2, 4]}
df = pd.DataFrame(data=d)
a = Initializer(df)
self.assertIs(type(a), DataFrameInitializer)
self.assertFalse(a.constant())
self.assertFalse(a.verified)
self.assertTrue(a.contains_indices())
self.assertEqual(list(a.indices()), [0, 1, 2])
self.assertEqual(a(None, 0), 1)
self.assertEqual(a(None, 1), 2)
self.assertEqual(a(None, 2), 4)
d = {'col1': [1, 2, 4], 'col2': [10, 20, 40]}
df = pd.DataFrame(data=d)
with self.assertRaisesRegex(
ValueError,
'DataFrameInitializer for DataFrame with multiple columns'):
a = Initializer(df)
a = DataFrameInitializer(df, 'col2')
self.assertIs(type(a), DataFrameInitializer)
self.assertFalse(a.constant())
self.assertFalse(a.verified)
self.assertTrue(a.contains_indices())
self.assertEqual(list(a.indices()), [0, 1, 2])
self.assertEqual(a(None, 0), 10)
self.assertEqual(a(None, 1), 20)
self.assertEqual(a(None, 2), 40)
df = pd.DataFrame([10, 20, 30, 40], index=[[0, 0, 1, 1], [0, 1, 0, 1]])
a = Initializer(df)
self.assertIs(type(a), DataFrameInitializer)
self.assertFalse(a.constant())
self.assertFalse(a.verified)
self.assertTrue(a.contains_indices())
self.assertEqual(list(a.indices()), [(0, 0), (0, 1), (1, 0), (1, 1)])
self.assertEqual(a(None, (0, 0)), 10)
self.assertEqual(a(None, (0, 1)), 20)
self.assertEqual(a(None, (1, 0)), 30)
self.assertEqual(a(None, (1, 1)), 40)
def test_diagnostic_mode(self):
self.pest.diagnostic_mode = True
objval, thetavals = self.pest.theta_est()
asym = np.arange(10, 30, 2)
rate = np.arange(0, 1.5, 0.25)
theta_vals = pd.DataFrame(list(product(asym, rate)),
columns=self.pest.theta_names)
obj_at_theta = self.pest.objective_at_theta(theta_vals)
self.pest.diagnostic_mode = False
def make_model(self, theta_names):
data = pd.DataFrame(data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0],
[5, 15.6], [7, 19.8]],
columns=['hour', 'y'])
def rooney_biegler_model_alternate(data):
''' Alternate model definition used in a unit test
Here, the fitted parameters are defined as a single variable over a set
A bit silly for this specific example
'''
model = pyo.ConcreteModel()
model.var_names = pyo.Set(
initialize=['asymptote', 'rate_constant'])
model.theta = pyo.Var(model.var_names,
initialize={
'asymptote': 15,
'rate_constant': 0.5
})
model.theta[
'asymptote'].fixed = True # parmest will unfix theta variables, even when they are indexed
model.theta['rate_constant'].fixed = True
def response_rule(m, h):
expr = m.theta['asymptote'] * (
1 - pyo.exp(-m.theta['rate_constant'] * h))
return expr
model.response_function = pyo.Expression(data.hour,
rule=response_rule)
def SSE_rule(m):
return sum((data.y[i] - m.response_function[data.hour[i]])**2
for i in data.index)
model.SSE = pyo.Objective(rule=SSE_rule, sense=pyo.minimize)
return model
def SSE(model, data):
expr = sum((data.y[i] - model.response_function[data.hour[i]])**2
for i in data.index)
return expr
return parmest.Estimator(rooney_biegler_model_alternate, data,
theta_names, SSE)
def setUp(self):
from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import rooney_biegler_model
data = pd.DataFrame(data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0],
[5, 15.6], [6, 19.8]],
columns=['hour', 'y'])
theta_names = ['asymptote', 'rate_constant']
def SSE(model, data):
expr = sum((data.y[i] - model.response_function[data.hour[i]])**2
for i in data.index)
return expr
self.pest = parmest.Estimator(rooney_biegler_model, data, theta_names,
SSE)
def test_ipopt_solve_with_stats(self):
from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import rooney_biegler_model
from pyomo.contrib.parmest.ipopt_solver_wrapper import ipopt_solve_with_stats
data = pd.DataFrame(data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0],
[5, 15.6], [7, 19.8]],
columns=['hour', 'y'])
model = rooney_biegler_model(data)
solver = pyo.SolverFactory('ipopt')
solver.solve(model)
status_obj, solved, iters, time, regu = ipopt_solve_with_stats(
model, solver)
self.assertEqual(solved, True)
def _simple_model(self, add_constraint=False):
# Hardwired to have two x columns and one y
# if add_constraint is true, there is a binding constraint on b0
data = pd.DataFrame([[1, 1.1, 0.365759306],
[2, 1.2, 4],
[3, 1.3, 4.8876684],
[4, 1.4, 5.173455561],
[5, 1.5, 2.093799081],
[6, 1.6, 9],
[7, 1.7, 6.475045106],
[8, 1.8, 8.127111268],
[9, 1.9, 6],
[10, 1.21, 10.20642714],
[11, 1.22, 13.08211636],
[12, 1.23, 10],
[13, 1.24, 15.38766047],
[14, 1.25, 14.6587746],
[15, 1.26, 13.68608604],
[16, 1.27, 14.70707893],
[17, 1.28, 18.46192779],
[18, 1.29, 15.60649164]],
columns=['tofu','chard', 'y'])
model = pyo.ConcreteModel()
model.b0 = pyo.Var(initialize = 0)
model.bindexes = pyo.Set(initialize=['tofu', 'chard'])
model.b = pyo.Var(model.bindexes, initialize = 1)
# try to make trouble
if add_constraint:
model.binding_constraint = pyo.Constraint(expr=model.b0>=10)
# The columns need to have unique values (or you get warnings)
def response_rule(m, t, c):
expr = m.b0 + m.b['tofu']*t + m.b['chard']*c
return expr
model.response_function = pyo.Expression(data.tofu, data.chard, rule = response_rule)
def SSE_rule(m):
return sum((data.y[i] - m.response_function[data.tofu[i], data.chard[i]])**2\
for i in data.index)
model.SSE = pyo.Objective(rule = SSE_rule, sense=pyo.minimize)
return model
def test_likelihood_ratio(self):
objval, thetavals = self.pest.theta_est()
asym = np.arange(10, 30, 2)
rate = np.arange(0, 1.5, 0.25)
theta_vals = pd.DataFrame(list(product(asym, rate)),
columns=self.pest.theta_names)
obj_at_theta = self.pest.objective_at_theta(theta_vals)
LR = self.pest.likelihood_ratio_test(obj_at_theta, objval,
[0.8, 0.9, 1.0])
self.assertTrue(set(LR.columns) >= set([0.8, 0.9, 1.0]))
self.assertTrue(LR[0.8].sum() == 6)
self.assertTrue(LR[0.9].sum() == 10)
self.assertTrue(LR[1.0].sum() == 60) # all true
graphics.pairwise_plot(LR, thetavals, 0.8)
def _get_data_slice(xvar,yvar,columns,data,theta_star):
search_ranges = {}
for var in columns:
if var in [xvar,yvar]:
search_ranges[var] = data[var].unique()
else:
search_ranges[var] = [theta_star[var]]
data_slice = pd.DataFrame(list(itertools.product(*search_ranges.values())),
columns=search_ranges.keys())
# griddata will not work with linear interpolation if the data
# values are constant in any dimension
for col in data[columns].columns:
cv = data[col].std()/data[col].mean() # Coefficient of variation
if cv < 1e-8:
temp = data.copy()
# Add variation (the interpolation is later scaled)
if cv == 0:
temp[col] = temp[col] + data[col].mean()/10
else:
temp[col] = temp[col] + data[col].std()
data = data.append(temp, ignore_index=True)
data_slice['obj'] = scipy.interpolate.griddata(
np.array(data[columns]),
np.array(data[['obj']]),
np.array(data_slice[columns]),
method='linear',
rescale=True,
)
X = data_slice[xvar]
Y = data_slice[yvar]
Z = data_slice['obj']
return X,Y,Z
def objective_at_theta(self, theta_values):
"""
Objective value for each theta
Parameters
----------
theta_values: DataFrame, columns=theta_names
Values of theta used to compute the objective
Returns
-------
obj_at_theta: DataFrame
Objective value for each theta (infeasible solutions are
omitted).
"""
assert isinstance(theta_values, pd.DataFrame)
# for parallel code we need to use lists and dicts in the loop
theta_names = theta_values.columns
all_thetas = theta_values.to_dict('records')
task_mgr = mpiu.ParallelTaskManager(len(all_thetas))
local_thetas = task_mgr.global_to_local_data(all_thetas)
# walk over the mesh, return objective function
all_obj = list()
for Theta in local_thetas:
obj, thetvals, worststatus = self._Q_at_theta(Theta)
if worststatus != pyo.TerminationCondition.infeasible:
all_obj.append(list(Theta.values()) + [obj])
# DLW, Aug2018: should we also store the worst solver status?
global_all_obj = task_mgr.allgather_global_data(all_obj)
dfcols = list(theta_names) + ['obj']
obj_at_theta = pd.DataFrame(data=global_all_obj, columns=dfcols)
return obj_at_theta
def theta_est_bootstrap(self, bootstrap_samples, samplesize=None,
replacement=True, seed=None, return_samples=False):
"""
Parameter estimation using bootstrap resampling of the data
Parameters
----------
bootstrap_samples: int
Number of bootstrap samples to draw from the data
samplesize: int or None, optional
Size of each bootstrap sample. If samplesize=None, samplesize will be
set to the number of samples in the data
replacement: bool, optional
Sample with or without replacement
seed: int or None, optional
Random seed
return_samples: bool, optional
Return a list of sample numbers used in each bootstrap estimation
Returns
-------
bootstrap_theta: DataFrame
Theta values for each sample and (if return_samples = True)
the sample numbers used in each estimation
"""
assert isinstance(bootstrap_samples, int)
assert isinstance(samplesize, (type(None), int))
assert isinstance(replacement, bool)
assert isinstance(seed, (type(None), int))
assert isinstance(return_samples, bool)
if samplesize is None:
samplesize = len(self._numbers_list)
if seed is not None:
np.random.seed(seed)
global_list = self._get_sample_list(samplesize, bootstrap_samples,
replacement)
task_mgr = mpiu.ParallelTaskManager(bootstrap_samples)
local_list = task_mgr.global_to_local_data(global_list)
# Reset numbers_list
self._numbers_list = list(range(samplesize))
bootstrap_theta = list()
for idx, sample in local_list:
objval, thetavals = self.theta_est(bootlist=list(sample))
thetavals['samples'] = sample
bootstrap_theta.append(thetavals)
# Reset numbers_list (back to original)
self._numbers_list = list(range(len(self.callback_data)))
global_bootstrap_theta = task_mgr.allgather_global_data(bootstrap_theta)
bootstrap_theta = pd.DataFrame(global_bootstrap_theta)
if not return_samples:
del bootstrap_theta['samples']
return bootstrap_theta
def _Q_opt(self,
ThetaVals=None,
solver="ef_ipopt",
return_values=[],
bootlist=None,
calc_cov=False):
"""
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
NOTE: If thetavals is present it will be attached to the
scenario tree so it can be used by the scenario creation
callback. Side note (feb 2018, dlw): if you later decide to
construct the tree just once and reuse it, then remember to
remove thetavals from it when none is desired.
"""
assert (solver != "k_aug" or ThetaVals == None)
# Create a tree with dummy scenarios (callback will supply when needed).
# Which names to use (i.e., numbers) depends on if it is for bootstrap.
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
tree_model = _treemaker(self._numbers_list)
else:
tree_model = _treemaker(range(len(self._numbers_list)))
stage1 = tree_model.Stages[1]
stage2 = tree_model.Stages[2]
tree_model.StageVariables[stage1] = self.theta_names
tree_model.StageVariables[stage2] = []
tree_model.StageCost[stage1] = "FirstStageCost"
tree_model.StageCost[stage2] = "SecondStageCost"
# Now attach things to the tree_model to pass them to the callback
tree_model.CallbackModule = None
tree_model.CallbackFunction = self._instance_creation_callback
if ThetaVals is not None:
tree_model.ThetaVals = ThetaVals
if bootlist is not None:
tree_model.BootList = bootlist
tree_model.cb_data = self.callback_data # None is OK
stsolver = st.StochSolver(fsfile="pyomo.contrib.parmest.parmest",
fsfct="_pysp_instance_creation_callback",
tree_model=tree_model)
# Solve the extensive form with ipopt
if solver == "ef_ipopt":
# Generate the extensive form of the stochastic program using pysp
self.ef_instance = stsolver.make_ef()
# need_gap is a holdover from solve_ef in rapper.py. Would we ever want
# need_gap = True with parmest?
need_gap = False
assert not (
need_gap and self.calc_cov
), "Calculating both the gap and reduced hessian (covariance) is not currently supported."
if not calc_cov:
# Do not calculate the reduced hessian
solver = SolverFactory('ipopt')
if self.solver_options is not None:
for key in self.solver_options:
solver.options[key] = self.solver_options[key]
if need_gap:
solve_result = solver.solve(self.ef_instance,
tee=self.tee,
load_solutions=False)
if len(solve_result.solution) > 0:
absgap = solve_result.solution(0).gap
else:
absgap = None
self.ef_instance.solutions.load_from(solve_result)
else:
solve_result = solver.solve(self.ef_instance, tee=self.tee)
elif not asl_available:
raise ImportError(
"parmest requires ASL to calculate the covariance matrix with solver 'ipopt'"
)
else:
# parmest makes the fitted parameters stage 1 variables
# thus we need to convert from var names (string) to
# Pyomo vars
ind_vars = []
for v in self.theta_names:
#ind_vars.append(eval('ef.'+v))
ind_vars.append(
self.ef_instance.MASTER_BLEND_VAR_RootNode[v])
# calculate the reduced hessian
solve_result, inv_red_hes = inv_reduced_hessian_barrier(
self.ef_instance,
independent_variables=ind_vars,
solver_options=self.solver_options,
tee=self.tee)
# Extract solution from pysp
stsolver.scenario_tree.pullScenarioSolutionsFromInstances()
stsolver.scenario_tree.snapshotSolutionFromScenarios(
) # update nodes
if self.diagnostic_mode:
print(' Solver termination condition = ',
str(solve_result.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for name, solval in stsolver.root_Var_solution():
thetavals[name] = solval
objval = stsolver.root_E_obj()
if calc_cov:
# Calculate the covariance matrix
# Extract number of data points considered
n = len(self.callback_data)
# Extract number of fitted parameters
l = len(thetavals)
# Assumption: Objective value is sum of squared errors
sse = objval
'''Calculate covariance assuming experimental observation errors are
independent and follow a Gaussian
distribution with constant variance.
The formula used in parmest was verified against equations (7-5-15) and
(7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974.
This formula is also applicable if the objective is scaled by a constant;
the constant cancels out. (PySP scaled by 1/n because it computes an
expected value.)
'''
cov = 2 * sse / (n - l) * inv_red_hes
if len(return_values) > 0:
var_values = []
for exp_i in self.ef_instance.component_objects(
Block, descend_into=False):
vals = {}
for var in return_values:
exp_i_var = eval('exp_i.' + str(var))
temp = [pyo.value(_) for _ in exp_i_var.itervalues()]
if len(temp) == 1:
vals[var] = temp[0]
else:
vals[var] = temp
var_values.append(vals)
var_values = pd.DataFrame(var_values)
if calc_cov:
return objval, thetavals, var_values, cov
else:
return objval, thetavals, var_values
if calc_cov:
return objval, thetavals, cov
else:
return objval, thetavals
# Solve with sipopt and k_aug
elif solver == "k_aug":
# Just hope for the best with respect to degrees of freedom.
model = stsolver.make_ef()
stream_solver = True
ipopt = SolverFactory('ipopt')
sipopt = SolverFactory('ipopt_sens')
kaug = SolverFactory('k_aug')
#: ipopt suffixes REQUIRED FOR K_AUG!
model.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT_EXPORT)
model.ipopt_zL_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zU_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zL_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.ipopt_zU_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
# declare the suffix to be imported by the solver
model.red_hessian = pyo.Suffix(direction=pyo.Suffix.EXPORT)
#: K_AUG SUFFIXES
model.dof_v = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.rh_name = pyo.Suffix(direction=pyo.Suffix.IMPORT)
for vstrindex in range(len(self.theta_names)):
vstr = self.theta_names[vstrindex]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
varobject.set_suffix_value(model.red_hessian, vstrindex + 1)
varobject.set_suffix_value(model.dof_v, 1)
#: rh_name will tell us which position the corresponding variable has on the reduced hessian text file.
#: be sure to declare the suffix value (order)
# dof_v is "degree of freedom variable"
kaug.options[
"compute_inv"] = "" #: if the reduced hessian is desired.
#: please check the inv_.in file if the compute_inv option was used
#: write some options for ipopt sens
with open('ipopt.opt', 'w') as f:
f.write('compute_red_hessian yes\n'
) #: computes the reduced hessian (sens_ipopt)
f.write('output_file my_ouput.txt\n')
f.write('rh_eigendecomp yes\n')
f.close()
#: Solve
sipopt.solve(model, tee=stream_solver)
with open('ipopt.opt', 'w') as f:
f.close()
ipopt.solve(model, tee=stream_solver)
model.ipopt_zL_in.update(model.ipopt_zL_out)
model.ipopt_zU_in.update(model.ipopt_zU_out)
#: k_aug
print('k_aug \n\n\n')
#m.write('problem.nl', format=ProblemFormat.nl)
kaug.solve(model, tee=stream_solver)
HessDict = {}
thetavals = {}
print('k_aug red_hess')
with open('result_red_hess.txt', 'r') as f:
lines = f.readlines()
# asseble the return values
objval = model.MASTER_OBJECTIVE_EXPRESSION.expr()
for i in range(len(lines)):
HessDict[self.theta_names[i]] = {}
linein = lines[i]
print(linein)
parts = linein.split()
for j in range(len(parts)):
HessDict[self.theta_names[i]][self.theta_names[j]] = \
float(parts[j])
# Get theta value (there is probably a better way...)
vstr = self.theta_names[i]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
thetavals[self.theta_names[i]] = pyo.value(varobject)
return objval, thetavals, HessDict
else:
raise RuntimeError("Unknown solver in Q_Opt=" + solver)
def _Q_opt(self,
ThetaVals=None,
solver="ef_ipopt",
return_values=[],
bootlist=None,
calc_cov=False,
cov_n=None):
"""
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
"""
if (solver == "k_aug"):
raise RuntimeError("k_aug no longer supported.")
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
senario_numbers = list(range(len(self.callback_data)))
scen_names = ["Scenario{}".format(i) for i in senario_numbers]
else:
scen_names = ["Scenario{}".format(i) for i in range(len(bootlist))]
# tree_model.CallbackModule = None
outer_cb_data = dict()
outer_cb_data["callback"] = self._instance_creation_callback
if ThetaVals is not None:
outer_cb_data["ThetaVals"] = ThetaVals
if bootlist is not None:
outer_cb_data["BootList"] = bootlist
outer_cb_data["cb_data"] = self.callback_data # None is OK
outer_cb_data["theta_names"] = self.theta_names
options = {"solver": "ipopt"}
scenario_creator_options = {"cb_data": outer_cb_data}
if use_mpisppy:
ef = sputils.create_EF(
scen_names,
_experiment_instance_creation_callback,
EF_name="_Q_opt",
suppress_warnings=True,
scenario_creator_kwargs=scenario_creator_options)
else:
ef = local_ef.create_EF(
scen_names,
_experiment_instance_creation_callback,
EF_name="_Q_opt",
suppress_warnings=True,
scenario_creator_kwargs=scenario_creator_options)
self.ef_instance = ef
# Solve the extensive form with ipopt
if solver == "ef_ipopt":
if not calc_cov:
# Do not calculate the reduced hessian
solver = SolverFactory('ipopt')
if self.solver_options is not None:
for key in self.solver_options:
solver.options[key] = self.solver_options[key]
solve_result = solver.solve(ef, tee=self.tee)
# The import error will be raised when we attempt to use
# inv_reduced_hessian_barrier below.
#
#elif not asl_available:
# raise ImportError("parmest requires ASL to calculate the "
# "covariance matrix with solver 'ipopt'")
else:
# parmest makes the fitted parameters stage 1 variables
ind_vars = []
for ndname, Var, solval in ef_nonants(ef):
ind_vars.append(Var)
# calculate the reduced hessian
solve_result, inv_red_hes = \
inverse_reduced_hessian.inv_reduced_hessian_barrier(
self.ef_instance,
independent_variables= ind_vars,
solver_options=self.solver_options,
tee=self.tee)
if self.diagnostic_mode:
print(' Solver termination condition = ',
str(solve_result.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for ndname, Var, solval in ef_nonants(ef):
# process the name
# the scenarios are blocks, so strip the scenario name
vname = Var.name[Var.name.find(".") + 1:]
thetavals[vname] = solval
objval = pyo.value(ef.EF_Obj)
if calc_cov:
# Calculate the covariance matrix
# Number of data points considered
n = cov_n
# Extract number of fitted parameters
l = len(thetavals)
# Assumption: Objective value is sum of squared errors
sse = objval
'''Calculate covariance assuming experimental observation errors are
independent and follow a Gaussian
distribution with constant variance.
The formula used in parmest was verified against equations (7-5-15) and
(7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974.
This formula is also applicable if the objective is scaled by a constant;
the constant cancels out. (was scaled by 1/n because it computes an
expected value.)
'''
cov = 2 * sse / (n - l) * inv_red_hes
cov = pd.DataFrame(cov,
index=thetavals.keys(),
columns=thetavals.keys())
thetavals = pd.Series(thetavals)
if len(return_values) > 0:
var_values = []
for exp_i in self.ef_instance.component_objects(
Block, descend_into=False):
vals = {}
for var in return_values:
exp_i_var = exp_i.find_component(str(var))
if exp_i_var is None: # we might have a block such as _mpisppy_data
continue
temp = [pyo.value(_) for _ in exp_i_var.values()]
if len(temp) == 1:
vals[var] = temp[0]
else:
vals[var] = temp
if len(vals) > 0:
var_values.append(vals)
var_values = pd.DataFrame(var_values)
if calc_cov:
return objval, thetavals, var_values, cov
else:
return objval, thetavals, var_values
if calc_cov:
return objval, thetavals, cov
else:
return objval, thetavals
else:
raise RuntimeError("Unknown solver in Q_Opt=" + solver)
def pairwise_plot(theta_values, theta_star=None, alpha=None, distributions=[],
axis_limits=None, title=None, add_obj_contour=True,
add_legend=True, filename=None):
"""
Plot pairwise relationship for theta values, and optionally alpha-level
confidence intervals and objective value contours
Parameters
----------
theta_values: DataFrame or tuple
* If theta_values is a DataFrame, then it contains one column for each theta variable
and (optionally) an objective value column ('obj') and columns that contains
Boolean results from confidence interval tests (labeled using the alpha value).
Each row is a sample.
* Theta variables can be computed from ``theta_est_bootstrap``,
``theta_est_leaveNout``, and ``leaveNout_bootstrap_test``.
* The objective value can be computed using the ``likelihood_ratio_test``.
* Results from confidence interval tests can be computed using the
``leaveNout_bootstrap_test``, ``likelihood_ratio_test``, and
``confidence_region_test``.
* If theta_values is a tuple, then it contains a mean, covariance, and number
of samples (mean, cov, n) where mean is a dictionary or Series
(indexed by variable name), covariance is a DataFrame (indexed by
variable name, one column per variable name), and n is an integer.
The mean and covariance are used to create a multivariate normal
sample of n theta values. The covariance can be computed using
``theta_est(calc_cov=True)``.
theta_star: dict or Series, optional
Estimated value of theta. The dictionary or Series is indexed by variable name.
Theta_star is used to slice higher dimensional contour intervals in 2D
alpha: float, optional
Confidence interval value, if an alpha value is given and the
distributions list is empty, the data will be filtered by True/False
values using the column name whose value equals alpha (see results from
``leaveNout_bootstrap_test``, ``likelihood_ratio_test``, and
``confidence_region_test``)
distributions: list of strings, optional
Statistical distribution used to define a confidence region,
options = 'MVN' for multivariate_normal, 'KDE' for gaussian_kde, and
'Rect' for rectangular.
Confidence interval is a 2D slice, using linear interpolation at theta_star.
axis_limits: dict, optional
Axis limits in the format {variable: [min, max]}
title: string, optional
Plot title
add_obj_contour: bool, optional
Add a contour plot using the column 'obj' in theta_values.
Contour plot is a 2D slice, using linear interpolation at theta_star.
add_legend: bool, optional
Add a legend to the plot
filename: string, optional
Filename used to save the figure
"""
assert isinstance(theta_values, (pd.DataFrame, tuple))
assert isinstance(theta_star, (type(None), dict, pd.Series, pd.DataFrame))
assert isinstance(alpha, (type(None), int, float))
assert isinstance(distributions, list)
assert set(distributions).issubset(set(['MVN', 'KDE', 'Rect']))
assert isinstance(axis_limits, (type(None), dict))
assert isinstance(title, (type(None), str))
assert isinstance(add_obj_contour, bool)
assert isinstance(filename, (type(None), str))
# If theta_values is a tuple containing (mean, cov, n), create a DataFrame of values
if isinstance(theta_values, tuple):
assert(len(theta_values) == 3)
mean = theta_values[0]
cov = theta_values[1]
n = theta_values[2]
if isinstance(mean, dict):
mean = pd.Series(mean)
theta_names = mean.index
mvn_dist = stats.multivariate_normal(mean, cov)
theta_values = pd.DataFrame(mvn_dist.rvs(n, random_state=1), columns=theta_names)
assert(theta_values.shape[0] > 0)
if isinstance(theta_star, dict):
theta_star = pd.Series(theta_star)
if isinstance(theta_star, pd.DataFrame):
theta_star = theta_star.loc[0,:]
theta_names = [col for col in theta_values.columns if (col not in ['obj'])
and (not isinstance(col, float)) and (not isinstance(col, int))]
# Filter data by alpha
if (alpha in theta_values.columns) and (len(distributions) == 0):
thetas = theta_values.loc[theta_values[alpha] == True, theta_names]
else:
thetas = theta_values[theta_names]
if theta_star is not None:
theta_star = theta_star[theta_names]
legend_elements = []
g = sns.PairGrid(thetas)
# Plot histogram on the diagonal
# Note: distplot is deprecated and will be removed in a future
# version of seaborn, use histplot. distplot is kept for older
# versions of python.
if check_min_version(sns, "0.11"):
g.map_diag(sns.histplot)
else:
g.map_diag(sns.distplot, kde=False, hist=True, norm_hist=False)
# Plot filled contours using all theta values based on obj
if 'obj' in theta_values.columns and add_obj_contour:
g.map_offdiag(_add_obj_contour, columns=theta_names, data=theta_values,
theta_star=theta_star)
# Plot thetas
g.map_offdiag(plt.scatter, s=10)
legend_elements.append(matplotlib.lines.Line2D(
[0], [0], marker='o', color='w', label='thetas',
markerfacecolor='cadetblue', markersize=5))
# Plot theta*
if theta_star is not None:
g.map_offdiag(_add_scatter, color='k', columns=theta_names, theta_star=theta_star)
legend_elements.append(matplotlib.lines.Line2D(
[0], [0], marker='o', color='w', label='theta*',
markerfacecolor='k', markersize=6))
# Plot confidence regions
colors = ['r', 'mediumblue', 'darkgray']
if (alpha is not None) and (len(distributions) > 0):
if theta_star is None:
print("""theta_star is not defined, confidence region slice will be
plotted at the mean value of theta""")
theta_star = thetas.mean()
mvn_dist = None
kde_dist = None
for i, dist in enumerate(distributions):
if dist == 'Rect':
lb, ub = fit_rect_dist(thetas, alpha)
g.map_offdiag(_add_rectangle_CI, color=colors[i], columns=theta_names,
lower_bound=lb, upper_bound=ub)
legend_elements.append(matplotlib.lines.Line2D(
[0], [0], color=colors[i], lw=1, label=dist))
elif dist == 'MVN':
mvn_dist = fit_mvn_dist(thetas)
Z = mvn_dist.pdf(thetas)
score = stats.scoreatpercentile(Z, (1-alpha)*100)
g.map_offdiag(_add_scipy_dist_CI, color=colors[i], columns=theta_names,
ncells=100, alpha=score, dist=mvn_dist,
theta_star=theta_star)
legend_elements.append(matplotlib.lines.Line2D(
[0], [0], color=colors[i], lw=1, label=dist))
elif dist == 'KDE':
kde_dist = fit_kde_dist(thetas)
Z = kde_dist.pdf(thetas.transpose())
score = stats.scoreatpercentile(Z, (1-alpha)*100)
g.map_offdiag(_add_scipy_dist_CI, color=colors[i], columns=theta_names,
ncells=100, alpha=score, dist=kde_dist,
theta_star=theta_star)
legend_elements.append(matplotlib.lines.Line2D(
[0], [0], color=colors[i], lw=1, label=dist))
_set_axis_limits(g, axis_limits, thetas, theta_star)
for ax in g.axes.flatten():
ax.ticklabel_format(style='sci', scilimits=(-2,2), axis='both')
if add_legend:
xvar, yvar, loc = _get_variables(ax, theta_names)
if loc == (len(theta_names)-1,0):
ax.legend(handles=legend_elements, loc='best', prop={'size': 8})
if title:
g.fig.subplots_adjust(top=0.9)
g.fig.suptitle(title)
# Work in progress
# Plot lower triangle graphics in separate figures, useful for presentations
lower_triangle_only = False
if lower_triangle_only:
for ax in g.axes.flatten():
xvar, yvar, (xloc, yloc) = _get_variables(ax, theta_names)
if xloc < yloc: # lower triangle
ax.remove()
ax.set_xlabel(xvar)
ax.set_ylabel(yvar)
fig = plt.figure()
ax.figure=fig
fig.axes.append(ax)
fig.add_axes(ax)
f, dummy = plt.subplots()
bbox = dummy.get_position()
ax.set_position(bbox)
dummy.remove()
plt.close(f)
ax.tick_params(reset=True)
if add_legend:
ax.legend(handles=legend_elements, loc='best', prop={'size': 8})
plt.close(g.fig)
if filename is None:
plt.show()
else:
plt.savefig(filename)
plt.close()
def _Q_opt(self, ThetaVals=None, solver="ef_ipopt",
return_values=[], bootlist=None):
"""
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
NOTE: If thetavals is present it will be attached to the
scenario tree so it can be used by the scenario creation
callback. Side note (feb 2018, dlw): if you later decide to
construct the tree just once and reuse it, then remember to
remove thetavals from it when none is desired.
"""
assert(solver != "k_aug" or ThetaVals == None)
# Create a tree with dummy scenarios (callback will supply when needed).
# Which names to use (i.e., numbers) depends on if it is for bootstrap.
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
tree_model = _treemaker(self._numbers_list)
else:
tree_model = _treemaker(range(len(self._numbers_list)))
stage1 = tree_model.Stages[1]
stage2 = tree_model.Stages[2]
tree_model.StageVariables[stage1] = self.theta_names
tree_model.StageVariables[stage2] = []
tree_model.StageCost[stage1] = "FirstStageCost"
tree_model.StageCost[stage2] = "SecondStageCost"
# Now attach things to the tree_model to pass them to the callback
tree_model.CallbackModule = None
tree_model.CallbackFunction = self._instance_creation_callback
if ThetaVals is not None:
tree_model.ThetaVals = ThetaVals
if bootlist is not None:
tree_model.BootList = bootlist
tree_model.cb_data = self.callback_data # None is OK
stsolver = st.StochSolver(fsfile = "pyomo.contrib.parmest.parmest",
fsfct = "_pysp_instance_creation_callback",
tree_model = tree_model)
if solver == "ef_ipopt":
ef_sol = stsolver.solve_ef('ipopt',
sopts=self.solver_options,
tee=self.tee)
if self.diagnostic_mode:
print(' Solver termination condition = ',
str(ef_sol.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for name, solval in stsolver.root_Var_solution():
thetavals[name] = solval
objval = stsolver.root_E_obj()
if len(return_values) > 0:
var_values = []
for exp_i in stsolver.ef_instance.component_objects(Block, descend_into=False):
vals = {}
for var in return_values:
exp_i_var = eval('exp_i.'+ str(var))
temp = [_.value for _ in exp_i_var.itervalues()]
if len(temp) == 1:
vals[var] = temp[0]
else:
vals[var] = temp
var_values.append(vals)
var_values = pd.DataFrame(var_values)
return objval, thetavals, var_values
return objval, thetavals
elif solver == "k_aug":
# Just hope for the best with respect to degrees of freedom.
model = stsolver.make_ef()
stream_solver = True
ipopt = SolverFactory('ipopt')
sipopt = SolverFactory('ipopt_sens')
kaug = SolverFactory('k_aug')
#: ipopt suffixes REQUIRED FOR K_AUG!
model.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT_EXPORT)
model.ipopt_zL_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zU_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zL_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.ipopt_zU_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
# declare the suffix to be imported by the solver
model.red_hessian = pyo.Suffix(direction=pyo.Suffix.EXPORT)
#: K_AUG SUFFIXES
model.dof_v = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.rh_name = pyo.Suffix(direction=pyo.Suffix.IMPORT)
for vstrindex in range(len(self.theta_names)):
vstr = self.theta_names[vstrindex]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
varobject.set_suffix_value(model.red_hessian, vstrindex+1)
varobject.set_suffix_value(model.dof_v, 1)
#: rh_name will tell us which position the corresponding variable has on the reduced hessian text file.
#: be sure to declare the suffix value (order)
# dof_v is "degree of freedom variable"
kaug.options["compute_inv"] = "" #: if the reduced hessian is desired.
#: please check the inv_.in file if the compute_inv option was used
#: write some options for ipopt sens
with open('ipopt.opt', 'w') as f:
f.write('compute_red_hessian yes\n') #: computes the reduced hessian (sens_ipopt)
f.write('output_file my_ouput.txt\n')
f.write('rh_eigendecomp yes\n')
f.close()
#: Solve
sipopt.solve(model, tee=stream_solver)
with open('ipopt.opt', 'w') as f:
f.close()
ipopt.solve(model, tee=stream_solver)
model.ipopt_zL_in.update(model.ipopt_zL_out)
model.ipopt_zU_in.update(model.ipopt_zU_out)
#: k_aug
print('k_aug \n\n\n')
#m.write('problem.nl', format=ProblemFormat.nl)
kaug.solve(model, tee=stream_solver)
HessDict = {}
thetavals = {}
print('k_aug red_hess')
with open('result_red_hess.txt', 'r') as f:
lines = f.readlines()
# asseble the return values
objval = model.MASTER_OBJECTIVE_EXPRESSION.expr()
for i in range(len(lines)):
HessDict[self.theta_names[i]] = {}
linein = lines[i]
print(linein)
parts = linein.split()
for j in range(len(parts)):
HessDict[self.theta_names[i]][self.theta_names[j]] = \
float(parts[j])
# Get theta value (there is probably a better way...)
vstr = self.theta_names[i]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
thetavals[self.theta_names[i]] = pyo.value(varobject)
return objval, thetavals, HessDict
else:
raise RuntimeError("Unknown solver in Q_Opt="+solver)
def setUp(self):
self.A = pd.DataFrame(np.random.randint(0,100,size=(100,4)), columns=list('ABCD'))
self.B = pd.DataFrame(np.random.randint(0,100,size=(100,4)), columns=list('ABCD'))
def setUp(self):
def ABC_model(data):
ca_meas = data['ca']
cb_meas = data['cb']
cc_meas = data['cc']
if isinstance(data, pd.DataFrame):
meas_t = data.index # time index
else: # dictionary
meas_t = list(ca_meas.keys()) # nested dictionary
ca0 = 1.0
cb0 = 0.0
cc0 = 0.0
m = pyo.ConcreteModel()
m.k1 = pyo.Var(initialize=0.5, bounds=(1e-4, 10))
m.k2 = pyo.Var(initialize=3.0, bounds=(1e-4, 10))
m.time = dae.ContinuousSet(bounds=(0.0, 5.0), initialize=meas_t)
# initialization and bounds
m.ca = pyo.Var(m.time, initialize=ca0, bounds=(-1e-3, ca0 + 1e-3))
m.cb = pyo.Var(m.time, initialize=cb0, bounds=(-1e-3, ca0 + 1e-3))
m.cc = pyo.Var(m.time, initialize=cc0, bounds=(-1e-3, ca0 + 1e-3))
m.dca = dae.DerivativeVar(m.ca, wrt=m.time)
m.dcb = dae.DerivativeVar(m.cb, wrt=m.time)
m.dcc = dae.DerivativeVar(m.cc, wrt=m.time)
def _dcarate(m, t):
if t == 0:
return pyo.Constraint.Skip
else:
return m.dca[t] == -m.k1 * m.ca[t]
m.dcarate = pyo.Constraint(m.time, rule=_dcarate)
def _dcbrate(m, t):
if t == 0:
return pyo.Constraint.Skip
else:
return m.dcb[t] == m.k1 * m.ca[t] - m.k2 * m.cb[t]
m.dcbrate = pyo.Constraint(m.time, rule=_dcbrate)
def _dccrate(m, t):
if t == 0:
return pyo.Constraint.Skip
else:
return m.dcc[t] == m.k2 * m.cb[t]
m.dccrate = pyo.Constraint(m.time, rule=_dccrate)
def ComputeFirstStageCost_rule(m):
return 0
m.FirstStageCost = pyo.Expression(rule=ComputeFirstStageCost_rule)
def ComputeSecondStageCost_rule(m):
return sum(
(m.ca[t] - ca_meas[t])**2 + (m.cb[t] - cb_meas[t])**2 +
(m.cc[t] - cc_meas[t])**2 for t in meas_t)
m.SecondStageCost = pyo.Expression(
rule=ComputeSecondStageCost_rule)
def total_cost_rule(model):
return model.FirstStageCost + model.SecondStageCost
m.Total_Cost_Objective = pyo.Objective(rule=total_cost_rule,
sense=pyo.minimize)
disc = pyo.TransformationFactory('dae.collocation')
disc.apply_to(m, nfe=20, ncp=2)
return m
# This example tests data formatted in 3 ways
# Each format holds 1 scenario
# 1. dataframe with time index
# 2. nested dictionary {ca: {t, val pairs}, ... }
data = [[0.000, 0.957, -0.031, -0.015], [0.263, 0.557, 0.330, 0.044],
[0.526, 0.342, 0.512, 0.156], [0.789, 0.224, 0.499, 0.310],
[1.053, 0.123, 0.428, 0.454], [1.316, 0.079, 0.396, 0.556],
[1.579, 0.035, 0.303, 0.651], [1.842, 0.029, 0.287, 0.658],
[2.105, 0.025, 0.221, 0.750], [2.368, 0.017, 0.148, 0.854],
[2.632, -0.002, 0.182, 0.845], [2.895, 0.009, 0.116, 0.893],
[3.158, -0.023, 0.079, 0.942], [3.421, 0.006, 0.078, 0.899],
[3.684, 0.016, 0.059, 0.942], [3.947, 0.014, 0.036, 0.991],
[4.211, -0.009, 0.014, 0.988], [4.474, -0.030, 0.036, 0.941],
[4.737, 0.004, 0.036, 0.971], [5.000, -0.024, 0.028, 0.985]]
data = pd.DataFrame(data, columns=['t', 'ca', 'cb', 'cc'])
data_df = data.set_index('t')
data_dict = {
'ca': {k: v
for (k, v) in zip(data.t, data.ca)},
'cb': {k: v
for (k, v) in zip(data.t, data.cb)},
'cc': {k: v
for (k, v) in zip(data.t, data.cc)}
}
theta_names = ['k1', 'k2']
self.pest_df = parmest.Estimator(ABC_model, [data_df], theta_names)
self.pest_dict = parmest.Estimator(ABC_model, [data_dict], theta_names)
# Create an instance of the model
self.m_df = ABC_model(data_df)
self.m_dict = ABC_model(data_dict)