def main():
scenario_creator = farmer.scenario_creator
CropsMult = int(sys.argv[1])
scen_count = int(sys.argv[2])
solver_name = sys.argv[3]
cb_data = {'use_integer': False, "CropsMult": CropsMult}
scenario_names = ['Scenario' + str(i) for i in range(scen_count)]
ef = sputils.create_EF(scenario_names,
scenario_creator,
creator_options={'cb_data': cb_data})
solver = pyo.SolverFactory(solver_name)
if 'persistent' in solver_name:
solver.set_instance(ef, symbolic_solver_labels=True)
solver.solve(tee=True)
else:
solver.solve(
ef,
tee=True,
symbolic_solver_labels=True,
)
print(f"EF objective: {pyo.value(ef.EF_Obj)}")
def main():
scenario_creator = farmer.scenario_creator
crops_multiplier = int(sys.argv[1])
scen_count = int(sys.argv[2])
solver_name = sys.argv[3]
scenario_creator_kwargs = {
"use_integer": False,
"crops_multiplier": crops_multiplier,
}
scenario_names = ['Scenario' + str(i) for i in range(scen_count)]
ef = sputils.create_EF(
scenario_names,
scenario_creator,
scenario_creator_kwargs=scenario_creator_kwargs,
)
solver = pyo.SolverFactory(solver_name)
if 'persistent' in solver_name:
solver.set_instance(ef, symbolic_solver_labels=True)
solver.solve(tee=True)
else:
solver.solve(
ef,
tee=True,
symbolic_solver_labels=True,
)
return ef
def __init__(self,
options,
all_scenario_names,
scenario_creator,
model_name=None,
scenario_creator_options=None,
suppress_warnings=False):
cb_data = None
if scenario_creator_options is not None:
if "cb_data" in scenario_creator_options:
cb_data = scenario_creator_options["cb_data"]
super().__init__(options,
all_scenario_names,
scenario_creator,
cb_data=cb_data)
if self.n_proc > 1 and self.rank == self.rank0:
logger.warning(
"Creating an ExtensiveForm object in parallel. Why?")
if scenario_creator_options is None:
scenario_creator_options = dict()
required = ["solver"]
self._options_check(required, self.options)
self.solver = pyo.SolverFactory(self.options["solver"])
self.ef = create_EF(
all_scenario_names,
scenario_creator,
creator_options=scenario_creator_options,
EF_name=model_name,
suppress_warnings=suppress_warnings,
)
def post_everything(self, PHIter, conv):
if (self.fixer_object):
self.fixer_object.post_everything(PHIter, conv)
self.reader_object.post_everything(PHIter, conv)
self.writer_object.post_everything(PHIter, conv)
# Custom fixer post-processing
if (self.rank != 0):
return
if (not self.fixer_object):
return
print('Fixed', self.fixer_object.fixed_so_far, 'unique variables')
# Solve the resulting EF with all variables fixed
scenario_names = self.ph.all_scenario_names
scenario_creator = self.ph.scenario_creator
ef = create_EF(scenario_names, scenario_creator, creator_options=None)
# Pick out any random scenario
sname = list(self.ph.local_scenarios.keys())[0]
scenario = self.ph.local_scenarios[sname]
nonant_vars = {
var.name: var
for node in scenario._PySPnode_list
for var in node.nonant_vardata_list
}
# Fix all the nonant vars in the EF
for (pair, var) in ef.ref_vars.items():
vname = extract_varname(var.name)
if nonant_vars[vname].is_fixed():
var.fix(nonant_vars[vname].value)
solver = pyo.SolverFactory('gurobi')
solver.solve(ef, tee=True)
def create_extensive_form(self):
self.creator_options = {
"cb_data": self.cb_data,
"md_dict": self.md_dict,
"first_time": 0,
"relaxation": self.relaxation
}
self.scenario_names = [
"Scenario_" + str(i) for i in range(
1,
len(self.cb_data["etree"].rootnode.ScenarioList) + 1)
]
self.ef = sputils.create_EF(self.scenario_names, util.pysp2_callback,
self.creator_options)
return self.ef
lines = list()
for this_branch in md_dict.elements("branch"):
lines.append(this_branch[0])
acstream = np.random.RandomState()
cb_data["etree"] = ET.ACTree(number_of_stages, branching_factors, seed,
acstream, a_line_fails_prob,
stage_duration_minutes, repair_fct, lines)
cb_data["epath"] = egret_path_to_data
cb_data["acstream"] = acstream
creator_options = {"cb_data": cb_data}
scenario_names=["Scenario_"+str(i)\
for i in range(1,len(cb_data["etree"].rootnode.ScenarioList)+1)]
# end options
ef = sputils.create_EF(scenario_names, pysp2_callback, creator_options)
###solver.options["BarHomogeneous"] = 1
if "gurobi" in solvername:
solver.options["BarHomogeneous"] = 1
results = solver.solve(ef, tee=True)
pyo.assert_optimal_termination(results)
print('EF objective value:', pyo.value(ef.EF_Obj))
sputils.ef_nonants_csv(ef, "vardump.csv")
for (sname, smodel) in sputils.ef_scenarios(ef):
print(sname)
for stage in smodel.stage_models:
print(" Stage {}".format(stage))
for gen in smodel.stage_models[stage].pg:
print (" gen={} pg={}, qg={}"\
BFs = options["branching_factors"]
ScenCount = BFs[0] * BFs[1]
all_scenario_names = list()
for sn in range(ScenCount):
all_scenario_names.append("Scen" + str(sn + 1))
# end hardwire
# This is multi-stage, so we need to supply node names
all_nodenames = sputils.create_nodenames_from_BFs(BFs)
# **** ef ****
solver = pyo.SolverFactory(options["solvername"])
ef = sputils.create_EF(
all_scenario_names,
scenario_creator,
scenario_creator_kwargs={"branching_factors": BFs},
)
results = solver.solve(ef, tee=options["verbose"])
print('EF objective value:', pyo.value(ef.EF_Obj))
sputils.ef_nonants_csv(ef, "vardump.csv")
# **** ph ****
options["xhat_specific_options"] = {"xhat_solver_options":
options["iterk_solver_options"],
"xhat_scenario_dict": \
{"ROOT": "Scen1",
"ROOT_0": "Scen1",
"ROOT_1": "Scen4",
"ROOT_2": "Scen7"},
"csvname": "specific.csv"}
def run(self):
""" Top-level execution."""
if self.is_EF:
ef = sputils.create_EF(
self.scenario_names,
self.scenario_creator,
scenario_creator_kwargs=self.kwargs,
suppress_warnings=True,
)
solvername = self.solvername
solver = pyo.SolverFactory(solvername)
if hasattr(self, "solver_options") and (self.solver_options
is not None):
for option_key, option_value in self.solver_options.items():
if option_value is not None:
solver.options[option_key] = option_value
if self.verbose:
global_toc("Starting EF solve")
if 'persistent' in solvername:
solver.set_instance(ef, symbolic_solver_labels=True)
results = solver.solve(tee=False)
else:
results = solver.solve(
ef,
tee=False,
symbolic_solver_labels=True,
)
if self.verbose:
global_toc("Completed EF solve")
self.EF_Obj = pyo.value(ef.EF_Obj)
objs = sputils.get_objs(ef)
self.is_minimizing = objs[0].is_minimizing
#TBD : Write a function doing this
if self.is_minimizing:
self.best_outer_bound = results.Problem[0]['Lower bound']
self.best_inner_bound = results.Problem[0]['Upper bound']
else:
self.best_inner_bound = results.Problem[0]['Upper bound']
self.best_outer_bound = results.Problem[0]['Lower bound']
self.ef = ef
if 'write_solution' in self.options:
if 'first_stage_solution' in self.options['write_solution']:
sputils.write_ef_first_stage_solution(
self.ef,
self.options['write_solution']['first_stage_solution'])
if 'tree_solution' in self.options['write_solution']:
sputils.write_ef_tree_solution(
self.ef,
self.options['write_solution']['tree_solution'])
self.xhats = sputils.nonant_cache_from_ef(ef)
self.local_xhats = self.xhats #Every scenario is local for EF
self.first_stage_solution = {"ROOT": self.xhats["ROOT"]}
else:
self.ef = None
args = argparse.Namespace(**self.options)
#Create a hub dict
hub_name = find_hub(self.options['cylinders'], self.is_multi)
hub_creator = getattr(vanilla, hub_name + '_hub')
beans = {
"args": args,
"scenario_creator": self.scenario_creator,
"scenario_denouement": self.scenario_denouement,
"all_scenario_names": self.scenario_names,
"scenario_creator_kwargs": self.kwargs
}
if self.is_multi:
beans["all_nodenames"] = self.options["all_nodenames"]
hub_dict = hub_creator(**beans)
#Add extensions
if 'extensions' in self.options:
for extension in self.options['extensions']:
extension_creator = getattr(vanilla, 'add_' + extension)
hub_dict = extension_creator(hub_dict, args)
#Create spoke dicts
potential_spokes = find_spokes(self.options['cylinders'],
self.is_multi)
#We only use the spokes with an associated command line arg set to True
spokes = [
spoke for spoke in potential_spokes
if self.options['with_' + spoke]
]
list_of_spoke_dict = list()
for spoke in spokes:
spoke_creator = getattr(vanilla, spoke + '_spoke')
spoke_beans = copy.deepcopy(beans)
if spoke == "xhatspecific":
spoke_beans["scenario_dict"] = self.options[
"scenario_dict"]
spoke_dict = spoke_creator(**spoke_beans)
list_of_spoke_dict.append(spoke_dict)
spcomm, opt_dict = sputils.spin_the_wheel(hub_dict,
list_of_spoke_dict)
self.opt = spcomm.opt
self.cylinder_rank = self.opt.cylinder_rank
self.on_hub = ("hub_class" in opt_dict)
if self.on_hub: # we are on a hub rank
self.best_inner_bound = spcomm.BestInnerBound
self.best_outer_bound = spcomm.BestOuterBound
#NOTE: We do not get bounds on every rank, only on hub
# This should change if we want to use cylinders for MMW
if 'write_solution' in self.options:
if 'first_stage_solution' in self.options['write_solution']:
sputils.write_spin_the_wheel_first_stage_solution(
spcomm, opt_dict,
self.options['write_solution']['first_stage_solution'])
if 'tree_solution' in self.options['write_solution']:
sputils.write_spin_the_wheel_tree_solution(
spcomm, opt_dict,
self.options['write_solution']['tree_solution'])
if self.on_hub: #we are on a hub rank
a_sname = self.opt.local_scenario_names[0]
root = self.opt.local_scenarios[a_sname]._mpisppy_node_list[0]
self.first_stage_solution = {
"ROOT":
[pyo.value(var) for var in root.nonant_vardata_list]
}
self.local_xhats = sputils.local_nonant_cache(spcomm)
def _create_master_with_scenarios(self):
ef_scenarios = self.master_scenarios
## we want the correct probabilities to be set when
## calling create_EF
if len(ef_scenarios) > 1:
def scenario_creator_wrapper(name, **creator_options):
scenario = self.scenario_creator(name, **creator_options)
if not hasattr(scenario, 'PySP_prob'):
scenario.PySP_prob = 1. / len(self.all_scenario_names)
return scenario
master = sputils.create_EF(ef_scenarios,
scenario_creator_wrapper,
creator_options={
'node_names': None,
'cb_data': self.cb_data
})
nonant_list, nonant_ids = _get_nonant_ids_EF(master)
else:
master = self.scenario_creator(ef_scenarios[0],
node_names=None,
cb_data=self.cb_data)
if not hasattr(master, 'PySP_prob'):
master.PySP_prob = 1. / len(self.all_scenario_names)
nonant_list, nonant_ids = _get_nonant_ids(master)
self.master_vars = nonant_list
# creates the eta variables for scenarios that are NOT selected to be
# included in the master problem
eta_indx = [
scenario_name for scenario_name in self.all_scenario_names
if scenario_name not in self.master_scenarios
]
self._add_master_etas(master, eta_indx)
obj = find_active_objective(master)
repn = generate_standard_repn(obj.expr, quadratic=True)
if len(repn.nonlinear_vars) > 0:
raise ValueError(
"LShaped does not support models with nonlinear objective functions"
)
linear_vars = list(repn.linear_vars)
linear_coefs = list(repn.linear_coefs)
quadratic_coefs = list(repn.quadratic_coefs)
# adjust coefficients by scenario/bundle probability
scen_prob = master.PySP_prob
for i, var in enumerate(repn.linear_vars):
if id(var) not in nonant_ids:
linear_coefs[i] *= scen_prob
for i, (x, y) in enumerate(repn.quadratic_vars):
# only multiply through once
if id(x) not in nonant_ids:
quadratic_coefs[i] *= scen_prob
elif id(y) not in nonant_ids:
quadratic_coefs[i] *= scen_prob
# NOTE: the LShaped code negates the objective, so
# we do the same here for consistency
if not self.is_minimizing:
for i, coef in enumerate(linear_coefs):
linear_coefs[i] = -coef
for i, coef in enumerate(quadratic_coefs):
quadratic_coefs[i] = -coef
# add the etas
for var in master.eta.values():
linear_vars.append(var)
linear_coefs.append(1)
expr = LinearExpression(constant=repn.constant,
linear_coefs=linear_coefs,
linear_vars=linear_vars)
if repn.quadratic_vars:
expr += pyo.quicksum(
(coef * x * y
for coef, (x,
y) in zip(quadratic_coefs, repn.quadratic_vars)))
master.del_component(obj)
# set master objective function
master.obj = pyo.Objective(expr=expr, sense=pyo.minimize)
self.master = master
all_scenario_names.append("Scen"+str(sn+1))
# end hardwire
# This is multi-stage, so we need to supply node names
all_nodenames = ["ROOT"] # all trees must have this node
# The rest is a naming convention invented for this problem.
# Note that mpisppy does not have nodes at the leaves,
# and node names must end in a serial number.
for b in range(BFs[0]):
all_nodenames.append("ROOT_"+str(b))
# **** ef ****
solver = pyo.SolverFactory(PHoptions["solvername"])
ef = sputils.create_EF(all_scenario_names,
scenario_creator,
creator_options={"cb_data": BFs})
results = solver.solve(ef, tee=PHoptions["verbose"])
print('EF objective value:', pyo.value(ef.EF_Obj))
sputils.ef_nonants_csv(ef, "vardump.csv")
# **** ph ****
PHoptions["xhat_specific_options"] = {"xhat_solver_options":
PHoptions["iterk_solver_options"],
"xhat_scenario_dict": \
{"ROOT": "Scen1",
"ROOT_0": "Scen1",
"ROOT_1": "Scen4",
"ROOT_2": "Scen7"},
"csvname": "specific.csv"}
'verbose': False,
'display_progress': True,
'display_timing': False,
'iter0_solver_options': None,
'iterk_solver_options': None
}
ph = PH(
options=phoptions,
all_scenario_names=farmer.all_scenario_names,
scenario_creator=farmer.scenario_creator,
scenario_denouement=farmer.scenario_denouement,
)
ph.ph_main()
ef = sputils.create_EF(farmer.all_scenario_names, farmer.scenario_creator)
solver = pyo.SolverFactory(solver_name)
if 'persistent' in solver_name:
solver.set_instance(ef, symbolic_solver_labels=True)
solver.solve(tee=True)
else:
solver.solve(
ef,
tee=True,
symbolic_solver_labels=True,
)
print(f"EF objective: {pyo.value(ef.EF_Obj)}")
farmer.close()
def _Q_opt(self,
ThetaVals=None,
solver="ef_ipopt",
return_values=[],
bootlist=None,
calc_cov=False,
cov_n=None):
"""
Set up all thetas as first stage Vars, return resulting theta
values as well as the objective function value.
"""
if (solver == "k_aug"):
raise RuntimeError("k_aug no longer supported.")
# (Bootstrap scenarios will use indirection through the bootlist)
if bootlist is None:
senario_numbers = list(range(len(self.callback_data)))
scen_names = ["Scenario{}".format(i) for i in senario_numbers]
else:
scen_names = ["Scenario{}".format(i) for i in range(len(bootlist))]
# tree_model.CallbackModule = None
outer_cb_data = dict()
outer_cb_data["callback"] = self._instance_creation_callback
if ThetaVals is not None:
outer_cb_data["ThetaVals"] = ThetaVals
if bootlist is not None:
outer_cb_data["BootList"] = bootlist
outer_cb_data["cb_data"] = self.callback_data # None is OK
outer_cb_data["theta_names"] = self.theta_names
options = {"solver": "ipopt"}
scenario_creator_options = {"cb_data": outer_cb_data}
if use_mpisppy:
ef = sputils.create_EF(
scen_names,
_experiment_instance_creation_callback,
EF_name="_Q_opt",
suppress_warnings=True,
scenario_creator_kwargs=scenario_creator_options)
else:
ef = local_ef.create_EF(
scen_names,
_experiment_instance_creation_callback,
EF_name="_Q_opt",
suppress_warnings=True,
scenario_creator_kwargs=scenario_creator_options)
self.ef_instance = ef
# Solve the extensive form with ipopt
if solver == "ef_ipopt":
if not calc_cov:
# Do not calculate the reduced hessian
solver = SolverFactory('ipopt')
if self.solver_options is not None:
for key in self.solver_options:
solver.options[key] = self.solver_options[key]
solve_result = solver.solve(ef, tee=self.tee)
# The import error will be raised when we attempt to use
# inv_reduced_hessian_barrier below.
#
#elif not asl_available:
# raise ImportError("parmest requires ASL to calculate the "
# "covariance matrix with solver 'ipopt'")
else:
# parmest makes the fitted parameters stage 1 variables
ind_vars = []
for ndname, Var, solval in ef_nonants(ef):
ind_vars.append(Var)
# calculate the reduced hessian
solve_result, inv_red_hes = \
inverse_reduced_hessian.inv_reduced_hessian_barrier(
self.ef_instance,
independent_variables= ind_vars,
solver_options=self.solver_options,
tee=self.tee)
if self.diagnostic_mode:
print(' Solver termination condition = ',
str(solve_result.solver.termination_condition))
# assume all first stage are thetas...
thetavals = {}
for ndname, Var, solval in ef_nonants(ef):
# process the name
# the scenarios are blocks, so strip the scenario name
vname = Var.name[Var.name.find(".") + 1:]
thetavals[vname] = solval
objval = pyo.value(ef.EF_Obj)
if calc_cov:
# Calculate the covariance matrix
# Number of data points considered
n = cov_n
# Extract number of fitted parameters
l = len(thetavals)
# Assumption: Objective value is sum of squared errors
sse = objval
'''Calculate covariance assuming experimental observation errors are
independent and follow a Gaussian
distribution with constant variance.
The formula used in parmest was verified against equations (7-5-15) and
(7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974.
This formula is also applicable if the objective is scaled by a constant;
the constant cancels out. (was scaled by 1/n because it computes an
expected value.)
'''
cov = 2 * sse / (n - l) * inv_red_hes
cov = pd.DataFrame(cov,
index=thetavals.keys(),
columns=thetavals.keys())
thetavals = pd.Series(thetavals)
if len(return_values) > 0:
var_values = []
for exp_i in self.ef_instance.component_objects(
Block, descend_into=False):
vals = {}
for var in return_values:
exp_i_var = exp_i.find_component(str(var))
if exp_i_var is None: # we might have a block such as _mpisppy_data
continue
temp = [pyo.value(_) for _ in exp_i_var.values()]
if len(temp) == 1:
vals[var] = temp[0]
else:
vals[var] = temp
if len(vals) > 0:
var_values.append(vals)
var_values = pd.DataFrame(var_values)
if calc_cov:
return objval, thetavals, var_values, cov
else:
return objval, thetavals, var_values
if calc_cov:
return objval, thetavals, cov
else:
return objval, thetavals
else:
raise RuntimeError("Unknown solver in Q_Opt=" + solver)
for this_branch in md_dict.elements("branch"):
lines.append(this_branch[0])
acstream = np.random.RandomState()
scenario_creator_kwargs["etree"] = ET.ACTree(number_of_stages,
branching_factors, seed,
acstream, a_line_fails_prob,
stage_duration_minutes,
repair_fct, lines)
scenario_creator_kwargs["acstream"] = acstream
scenario_names=["Scenario_"+str(i)\
for i in range(1,len(scenario_creator_kwargs["etree"].rootnode.ScenarioList)+1)]
# end options
ef = sputils.create_EF(scenario_names,
pysp2_callback,
scenario_creator_kwargs=scenario_creator_kwargs)
###solver.options["BarHomogeneous"] = 1
if "gurobi" in solvername:
solver.options["BarHomogeneous"] = 1
results = solver.solve(ef, tee=True)
pyo.assert_optimal_termination(results)
print('EF objective value:', pyo.value(ef.EF_Obj))
sputils.ef_nonants_csv(ef, "vardump.csv")
for (sname, smodel) in sputils.ef_scenarios(ef):
print(sname)
for stage in smodel.stage_models:
print(" Stage {}".format(stage))
for gen in smodel.stage_models[stage].pg:
solvername = ama_object.options['EF_solver_name']
scenario_creator_kwargs = {
"use_integer": False,
"crops_multiplier": crops_multiplier,
"num_scens": ScenCount
}
# different scenarios for xhat
scenario_names = [
'Scenario' + str(i) for i in range(ScenCount, 2 * ScenCount)
]
ef = sputils.create_EF(
scenario_names,
scenario_creator,
scenario_creator_kwargs=scenario_creator_kwargs,
)
solver = pyo.SolverFactory(solvername)
if 'persistent' in solvername:
solver.set_instance(ef, symbolic_solver_labels=True)
solver.solve(tee=False)
else:
solver.solve(
ef,
tee=False,
symbolic_solver_labels=True,
)
print(f"Xhat in-sample objective: {pyo.value(ef.EF_Obj)}")
