def run(self):
"""
This method execute the optimization on the knapsack problem.
@ In, None
@ Out, None
"""
outputDict = {}
if self.uncertainties is None:
inputData = self.generateModelInputData()
# specifying the path to a solver
# with SolverFactory(self.solver, executable=self.executable) as opt:
with SolverFactory(self.solver) as opt:
opt.options.update(self.sopts) # add solver options
model = self.createInstance(inputData)
results = opt.solve(model, load_solutions=False, tee=self.tee, **{'use_signal_handling':False})
if results.solver.termination_condition != TerminationCondition.optimal:
raise RuntimeError("Solver did not report optimality:\n%s" %(results.solver))
model.solutions.load_from(results)
outputDict.update(self.printSolution(model))
self.output.update(outputDict)
# TODO: Add collect output and return a dictionary for raven to retrieve information
else:
tree_model = self.pysp_scenario_tree_model_callback()
if self.stochSolver == 'ef':
# fsfct the callback function for pysp_instance_creation_callback
# tree_model generated by pysp_scenario_tree_model_callback
stsolver = rapper.StochSolver("", fsfct=self.pysp_instance_creation_callback, tree_model=tree_model)
ef_sol = stsolver.solve_ef(self.solver, sopts=self.sopts, tee=self.tee)
if ef_sol.solver.termination_condition != TerminationCondition.optimal:
raise RuntimeError("Solver did not report optimality:\n%s" %(ef_sol.solver))
# TODO: Add collect output and return a dictionary for raven to retrieve information
self.printScenarioSolution(stsolver)
outputDict.update(self.getScenarioSolution(stsolver.scenario_tree))
self.output.update(outputDict)
elif self.stochSolver == 'ph':
# fsfct the callback function for pysp_instance_creation_callback
# tree_model generated by pysp_scenario_tree_model_callback
stsolver = rapper.StochSolver("", fsfct=self.pysp_instance_creation_callback, tree_model=tree_model, phopts=self.phopts)
ph_sol = stsolver.solve_ph(subsolver=self.solver, default_rho=self.phRho, phopts=self.phopts, sopts=self.sopts, tee=self.tee)
# TODO: Add collect output and return a dictionary for raven to retrieve information
# first retrieve the xhat solution from solver, then print the solution
obj, xhat = rapper.xhat_from_ph(ph_sol)
for nodeName, varName, varValue in rapper.xhat_walker(xhat):
print (nodeName, varName, varValue)
if ph_sol.solver.termination_condition != TerminationCondition.optimal:
raise RuntimeError("Solver did not report optimality:\n%s" %(ph_sol.solver))
self.printScenarioSolution(stsolver)
outputDict.update(self.getScenarioSolution(stsolver.scenario_tree))
self.output.update(outputDict)
return outputDict
def test_Abstract_Construction(self):
""" see if we can create the solver object for an AbstractModel"""
stsolver = rapper.StochSolver(self.farmer_ReferencePath,
fsfct = None,
tree_model = self.farmer_scenarioPath,
phopts = None)
def solve_sto(f, IndexReturn, Tbill):
#The next two lines show one way to create a concrete scenario tree. There are
#others that can be found in `pyomo.pysp.scenariotree.tree_structure_model`.
abstract_tree = CreateAbstractScenarioTreeModel()
concrete_tree = abstract_tree.create_instance("/Users/xiaoshiguo/Desktop/model"+str(J)+"/ScenarioStructure"+str(J)+".dat")
concrete_tree.IndexReturn = IndexReturn # line added by DLW
concrete_tree.Tbill = Tbill
stsolver = rapper.StochSolver("ReferenceModel"+str(J)+".py", fsfct = "pysp_instance_creation_callback", tree_model = concrete_tree)
#stsolver = rapper.StochSolver("/Users/xiaoshiguo/Desktop/portfolio/models/ReferenceModel.py", tree_model = concrete_tree)
ef_sol = stsolver.solve_ef('glpk', tee=False)
# ef_sol = stsolver.solve_ph(subsolver = solvername, default_rho = 1)
if ef_sol.solver.termination_condition != TerminationCondition.optimal:
print ("oops! not optimal:",ef_sol.solver.termination_condition)
#There is an iterator to loop over the root node solution:
for varname, varval in stsolver.root_Var_solution():
if varname == "weight":
weight = varval
else:
eta = varval
#print (varname, str(varval))
#a function to compute compute the objective function value
obj = stsolver.root_E_obj()
#print ("Expecatation take over scenarios=", obj)
# write down scenario tree in testcref file
#csvw.write_csv_soln(stsolver.scenario_tree, str(f))
return obj, weight, eta
def test_construct_default_tree_error(self):
"""verify that construction of concrete with default tree name gives error when it should"""
with self.assertRaises(AttributeError):
stsolver = rapper.StochSolver(
"ReferenceModel.py",
fsfct="pysp_instance_creation_callback",
tree_model=None)
def test_ef_cvar_construct(self):
""" construct the ef with cvar """
stsolver = rapper.StochSolver("ReferenceModel.py",
fsfct="pysp_instance_creation_callback",
tree_model=self.farmer_concrete_tree)
ef = stsolver.make_ef(generate_weighted_cvar=True,
cvar_weight=0.1,
risk_alpha=0.9)
def test_Abstract_ef(self):
""" see if we can create the solver object for an AbstractModel"""
stsolver = rapper.StochSolver(self.farmer_ReferencePath,
fsfct = None,
tree_model = self.farmer_scenarioPath,
phopts = None)
ef_sol = stsolver.solve_ef(solvername)
assert(ef_sol.solver.termination_condition \
== pyo.TerminationCondition.optimal)
def test_ef_solve_with_csvwriter(self):
""" solve the ef and report gap"""
stsolver = rapper.StochSolver("ReferenceModel.py",
fsfct="pysp_instance_creation_callback",
tree_model=self.farmer_concrete_tree)
res, gap = stsolver.solve_ef(solvername, tee=True, need_gap=True)
csvw.write_csv_soln(stsolver.scenario_tree, "testcref")
with open("testcref.csv", 'r') as f:
line = f.readline()
assert (line.split(",")[0] == "FirstStage")
def optimize_contract(path):
stsolver = rapper.StochSolver(path+'/ReferenceModel.py', fsfct = None, tree_model = path+'/ScenarioStructure.dat', phopts = None)
ef_sol = stsolver.solve_ef('cplex')
'''
while ef_sol.solver.termination_condition != pyo.TerminationCondition.optimal:
print('solving')
print(ef_sol.solver.termination_condition)
print('solved')
'''
return stsolver
def test_NetX_ef_csvwriter(self):
""" solve the ef and report gap"""
import NetXReferenceModel as ref
tree_model = ref.pysp_scenario_tree_model_callback()
stsolver = rapper.StochSolver("NetXReferenceModel.py",
fsfct="pysp_instance_creation_callback",
tree_model=tree_model)
res, gap = stsolver.solve_ef(solvername, tee=True, need_gap=True)
csvw.write_csv_soln(stsolver.scenario_tree, "testcref")
with open("testcref_StageCostDetail.csv", 'r') as f:
line = f.readline()
assert (line.split(",")[0] == "Stage1")
def test_ef_solve(self):
""" solve the ef and check some post solution code"""
stsolver = rapper.StochSolver("ReferenceModel.py",
tree_model=self.farmer_concrete_tree)
ef_sol = stsolver.solve_ef(solvername)
assert(ef_sol.solver.termination_condition \
== pyo.TerminationCondition.optimal)
for name, varval in stsolver.root_Var_solution():
#print (name, str(varval))
pass
self.assertAlmostEqual(varval, 170.0, 1)
obj = stsolver.root_E_obj()
def test_famer_netx(self):
""" solve the ef and check some post solution code"""
shutil.copyfile(self.farmpath + os.sep + "concreteNetX" +\
os.sep + "ReferenceModel.py",
self.tdir + os.sep + "ReferenceModel.py")
import ReferenceModel as RM
g = RM.pysp_scenario_tree_model_callback()
stsolver = rapper.StochSolver("ReferenceModel",
fsfct="pysp_instance_creation_callback",
tree_model=g)
ef_sol = stsolver.solve_ef(solvername)
assert(ef_sol.solver.termination_condition \
== pyo.TerminationCondition.optimal)
obj = stsolver.root_E_obj()
assert (abs(-108385 - obj) < 100) # any solver should get this close
def test_ph_solve(self):
""" use ph; assumes concrete two-stage json passes"""
phopts = {'--max-iterations': '2'}
stsolver = rapper.StochSolver("ReferenceModel.py",
tree_model=self.farmer_concrete_tree,
phopts=phopts)
ph = stsolver.solve_ph(subsolver=solvername,
default_rho=1,
phopts=phopts)
obj = stsolver.root_E_obj() # E[xbar]
obj, xhat = rapper.xhat_from_ph(ph)
for nodename, varname, varvalue in rapper.xhat_walker(xhat):
pass
assert (nodename == 'RootNode')
def test_fct_contruct(self):
""" give a callback function rather than a string"""
from ReferenceModel import pysp_instance_creation_callback
stsolver = rapper.StochSolver(None,
fsfct=pysp_instance_creation_callback,
tree_model=self.farmer_concrete_tree)
def Q_opt(self, ThetaVals=None, solver="ef_ipopt", bootlist=None):
"""
Mainly for internal use.
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
NOTE: If thetavals is present it will be attached to the
scenario tree so it can be used by the scenario creation
callback. Side note (feb 2018, dlw): if you later decide to
construct the tree just once and reuse it, then remember to
remove thetavals from it when none is desired.
Parameters
----------
ThetaVals: `dict`
A dictionary of theta values to fix in the pysp callback
(which has to grab them from the tree object and do the fixing)
solver: `string`
"ef_ipopt" or "k_aug".
Only ef is supported if ThetaVals is not None.
bootlist: `list` of `int`
The list is of scenario numbers for indirection used internally
by bootstrap.
The default is None and that is what driver authors should use.
Returns
-------
objectiveval: `float`
The objective function value
thetavals: `dict`
A dictionary of all values for theta
Hessian: `dict`
A dictionary of dictionaries for the Hessian.
The Hessian is not returned if the solver is ef.
"""
assert (solver != "k_aug" or ThetaVals == None)
# Create a tree with dummy scenarios (callback will supply when needed).
# Which names to use (i.e., numbers) depends on if it is for bootstrap.
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
tree_model = _treemaker(self.numbers_list)
else:
tree_model = _treemaker(range(len(self.numbers_list)))
stage1 = tree_model.Stages[1]
stage2 = tree_model.Stages[2]
tree_model.StageVariables[stage1] = self.thetalist
tree_model.StageVariables[stage2] = []
tree_model.StageCost[stage1] = "FirstStageCost"
tree_model.StageCost[stage2] = "SecondStageCost"
# Now attach things to the tree_model to pass them to the callback
tree_model.CallbackModule = self.gmodel_file
tree_model.CallbackFunction = self.gmodel_maker
if ThetaVals is not None:
tree_model.ThetaVals = ThetaVals
if bootlist is not None:
tree_model.BootList = bootlist
tree_model.cb_data = self.cb_data # None is OK
"""
stsolver = st.StochSolver(fsfile = self.gmodel_file,
fsfct = self.gmodel_maker,
tree_model = tree_model)
"""
stsolver = st.StochSolver(fsfile="pyomo.contrib.parmest.parmest",
fsfct="_pysp_instance_creation_callback",
tree_model=tree_model)
if solver == "ef_ipopt":
sopts = {}
sopts['max_iter'] = 6000
ef_sol = stsolver.solve_ef('ipopt', sopts=sopts, tee=self.tee)
if self.diagnostic_mode:
print(' solver termination condition=',
str(ef_sol.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for name, solval in stsolver.root_Var_solution():
thetavals[name] = solval
objval = stsolver.root_E_obj()
return objval, thetavals
elif solver == "k_aug":
# Just hope for the best with respect to degrees of freedom.
model = stsolver.make_ef()
stream_solver = True
ipopt = SolverFactory('ipopt')
sipopt = SolverFactory('ipopt_sens')
kaug = SolverFactory('k_aug')
#: ipopt suffixes REQUIRED FOR K_AUG!
model.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT_EXPORT)
model.ipopt_zL_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zU_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zL_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.ipopt_zU_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
# declare the suffix to be imported by the solver
model.red_hessian = pyo.Suffix(direction=pyo.Suffix.EXPORT)
#: K_AUG SUFFIXES
model.dof_v = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.rh_name = pyo.Suffix(direction=pyo.Suffix.IMPORT)
for vstrindex in range(len(self.thetalist)):
vstr = self.thetalist[vstrindex]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
varobject.set_suffix_value(model.red_hessian, vstrindex + 1)
varobject.set_suffix_value(model.dof_v, 1)
#: rh_name will tell us which position the corresponding variable has on the reduced hessian text file.
#: be sure to declare the suffix value (order)
# dof_v is "degree of freedom variable"
kaug.options[
"compute_inv"] = "" #: if the reduced hessian is desired.
#: please check the inv_.in file if the compute_inv option was used
#: write some options for ipopt sens
with open('ipopt.opt', 'w') as f:
f.write('compute_red_hessian yes\n'
) #: computes the reduced hessian (sens_ipopt)
f.write('output_file my_ouput.txt\n')
f.write('rh_eigendecomp yes\n')
f.close()
#: Solve
sipopt.solve(model, tee=stream_solver)
with open('ipopt.opt', 'w') as f:
f.close()
ipopt.solve(model, tee=stream_solver)
model.ipopt_zL_in.update(model.ipopt_zL_out)
model.ipopt_zU_in.update(model.ipopt_zU_out)
#: k_aug
print('k_aug \n\n\n')
#m.write('problem.nl', format=ProblemFormat.nl)
kaug.solve(model, tee=stream_solver)
HessDict = {}
thetavals = {}
print('k_aug red_hess')
with open('result_red_hess.txt', 'r') as f:
lines = f.readlines()
# asseble the return values
objval = model.MASTER_OBJECTIVE_EXPRESSION.expr()
for i in range(len(lines)):
HessDict[self.thetalist[i]] = {}
linein = lines[i]
print(linein)
parts = linein.split()
for j in range(len(parts)):
HessDict[self.thetalist[i]][self.thetalist[j]] = \
float(parts[j])
# Get theta value (there is probably a better way...)
vstr = self.thetalist[i]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
thetavals[self.thetalist[i]] = pyo.value(varobject)
return objval, thetavals, HessDict
else:
raise RuntimeError("Unknown solver in Q_Opt=" + solver)
def test_ef_solve_with_gap(self):
""" solve the ef and report gap"""
stsolver = rapper.StochSolver("ReferenceModel.py",
fsfct="pysp_instance_creation_callback",
tree_model=self.farmer_concrete_tree)
res, gap = stsolver.solve_ef(solvername, tee=True, need_gap=True)
def _Q_opt(self,
ThetaVals=None,
solver="ef_ipopt",
return_values=[],
bootlist=None,
calc_cov=False):
"""
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
NOTE: If thetavals is present it will be attached to the
scenario tree so it can be used by the scenario creation
callback. Side note (feb 2018, dlw): if you later decide to
construct the tree just once and reuse it, then remember to
remove thetavals from it when none is desired.
"""
assert (solver != "k_aug" or ThetaVals == None)
# Create a tree with dummy scenarios (callback will supply when needed).
# Which names to use (i.e., numbers) depends on if it is for bootstrap.
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
tree_model = _treemaker(self._numbers_list)
else:
tree_model = _treemaker(range(len(self._numbers_list)))
stage1 = tree_model.Stages[1]
stage2 = tree_model.Stages[2]
tree_model.StageVariables[stage1] = self.theta_names
tree_model.StageVariables[stage2] = []
tree_model.StageCost[stage1] = "FirstStageCost"
tree_model.StageCost[stage2] = "SecondStageCost"
# Now attach things to the tree_model to pass them to the callback
tree_model.CallbackModule = None
tree_model.CallbackFunction = self._instance_creation_callback
if ThetaVals is not None:
tree_model.ThetaVals = ThetaVals
if bootlist is not None:
tree_model.BootList = bootlist
tree_model.cb_data = self.callback_data # None is OK
stsolver = st.StochSolver(fsfile="pyomo.contrib.parmest.parmest",
fsfct="_pysp_instance_creation_callback",
tree_model=tree_model)
# Solve the extensive form with ipopt
if solver == "ef_ipopt":
# Generate the extensive form of the stochastic program using pysp
self.ef_instance = stsolver.make_ef()
# need_gap is a holdover from solve_ef in rapper.py. Would we ever want
# need_gap = True with parmest?
need_gap = False
assert not (
need_gap and self.calc_cov
), "Calculating both the gap and reduced hessian (covariance) is not currently supported."
if not calc_cov:
# Do not calculate the reduced hessian
solver = SolverFactory('ipopt')
if self.solver_options is not None:
for key in self.solver_options:
solver.options[key] = self.solver_options[key]
if need_gap:
solve_result = solver.solve(self.ef_instance,
tee=self.tee,
load_solutions=False)
if len(solve_result.solution) > 0:
absgap = solve_result.solution(0).gap
else:
absgap = None
self.ef_instance.solutions.load_from(solve_result)
else:
solve_result = solver.solve(self.ef_instance, tee=self.tee)
elif not asl_available:
raise ImportError(
"parmest requires ASL to calculate the covariance matrix with solver 'ipopt'"
)
else:
# parmest makes the fitted parameters stage 1 variables
# thus we need to convert from var names (string) to
# Pyomo vars
ind_vars = []
for v in self.theta_names:
#ind_vars.append(eval('ef.'+v))
ind_vars.append(
self.ef_instance.MASTER_BLEND_VAR_RootNode[v])
# calculate the reduced hessian
solve_result, inv_red_hes = inv_reduced_hessian_barrier(
self.ef_instance,
independent_variables=ind_vars,
solver_options=self.solver_options,
tee=self.tee)
# Extract solution from pysp
stsolver.scenario_tree.pullScenarioSolutionsFromInstances()
stsolver.scenario_tree.snapshotSolutionFromScenarios(
) # update nodes
if self.diagnostic_mode:
print(' Solver termination condition = ',
str(solve_result.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for name, solval in stsolver.root_Var_solution():
thetavals[name] = solval
objval = stsolver.root_E_obj()
if calc_cov:
# Calculate the covariance matrix
# Extract number of data points considered
n = len(self.callback_data)
# Extract number of fitted parameters
l = len(thetavals)
# Assumption: Objective value is sum of squared errors
sse = objval
'''Calculate covariance assuming experimental observation errors are
independent and follow a Gaussian
distribution with constant variance.
The formula used in parmest was verified against equations (7-5-15) and
(7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974.
This formula is also applicable if the objective is scaled by a constant;
the constant cancels out. (PySP scaled by 1/n because it computes an
expected value.)
'''
cov = 2 * sse / (n - l) * inv_red_hes
if len(return_values) > 0:
var_values = []
for exp_i in self.ef_instance.component_objects(
Block, descend_into=False):
vals = {}
for var in return_values:
exp_i_var = eval('exp_i.' + str(var))
temp = [pyo.value(_) for _ in exp_i_var.itervalues()]
if len(temp) == 1:
vals[var] = temp[0]
else:
vals[var] = temp
var_values.append(vals)
var_values = pd.DataFrame(var_values)
if calc_cov:
return objval, thetavals, var_values, cov
else:
return objval, thetavals, var_values
if calc_cov:
return objval, thetavals, cov
else:
return objval, thetavals
# Solve with sipopt and k_aug
elif solver == "k_aug":
# Just hope for the best with respect to degrees of freedom.
model = stsolver.make_ef()
stream_solver = True
ipopt = SolverFactory('ipopt')
sipopt = SolverFactory('ipopt_sens')
kaug = SolverFactory('k_aug')
#: ipopt suffixes REQUIRED FOR K_AUG!
model.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT_EXPORT)
model.ipopt_zL_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zU_out = pyo.Suffix(direction=pyo.Suffix.IMPORT)
model.ipopt_zL_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.ipopt_zU_in = pyo.Suffix(direction=pyo.Suffix.EXPORT)
# declare the suffix to be imported by the solver
model.red_hessian = pyo.Suffix(direction=pyo.Suffix.EXPORT)
#: K_AUG SUFFIXES
model.dof_v = pyo.Suffix(direction=pyo.Suffix.EXPORT)
model.rh_name = pyo.Suffix(direction=pyo.Suffix.IMPORT)
for vstrindex in range(len(self.theta_names)):
vstr = self.theta_names[vstrindex]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
varobject.set_suffix_value(model.red_hessian, vstrindex + 1)
varobject.set_suffix_value(model.dof_v, 1)
#: rh_name will tell us which position the corresponding variable has on the reduced hessian text file.
#: be sure to declare the suffix value (order)
# dof_v is "degree of freedom variable"
kaug.options[
"compute_inv"] = "" #: if the reduced hessian is desired.
#: please check the inv_.in file if the compute_inv option was used
#: write some options for ipopt sens
with open('ipopt.opt', 'w') as f:
f.write('compute_red_hessian yes\n'
) #: computes the reduced hessian (sens_ipopt)
f.write('output_file my_ouput.txt\n')
f.write('rh_eigendecomp yes\n')
f.close()
#: Solve
sipopt.solve(model, tee=stream_solver)
with open('ipopt.opt', 'w') as f:
f.close()
ipopt.solve(model, tee=stream_solver)
model.ipopt_zL_in.update(model.ipopt_zL_out)
model.ipopt_zU_in.update(model.ipopt_zU_out)
#: k_aug
print('k_aug \n\n\n')
#m.write('problem.nl', format=ProblemFormat.nl)
kaug.solve(model, tee=stream_solver)
HessDict = {}
thetavals = {}
print('k_aug red_hess')
with open('result_red_hess.txt', 'r') as f:
lines = f.readlines()
# asseble the return values
objval = model.MASTER_OBJECTIVE_EXPRESSION.expr()
for i in range(len(lines)):
HessDict[self.theta_names[i]] = {}
linein = lines[i]
print(linein)
parts = linein.split()
for j in range(len(parts)):
HessDict[self.theta_names[i]][self.theta_names[j]] = \
float(parts[j])
# Get theta value (there is probably a better way...)
vstr = self.theta_names[i]
varobject = _ef_ROOT_node_Object_from_string(model, vstr)
thetavals[self.theta_names[i]] = pyo.value(varobject)
return objval, thetavals, HessDict
else:
raise RuntimeError("Unknown solver in Q_Opt=" + solver)
def test_no_fsfct_no_tree(self):
"""verify that deprecated concrete with no fsfct is an error"""
with self.assertRaises(RuntimeError):
stsolver = rapper.StochSolver("ReferenceModel.py",
fsfct=None,
tree_model=None)
