def pysp_instance_creation_callback(scenario_name,
use_integer=False,
sense=pyo.minimize,
crops_multiplier=1):
# long function to create the entire model
# scenario_name is a string (e.g. AboveAverageScenario0)
#
# Returns a concrete model for the specified scenario
# scenarios come in groups of three
scengroupnum = sputils.extract_num(scenario_name)
scenario_base_name = scenario_name.rstrip("0123456789")
model = pyo.ConcreteModel()
def crops_init(m):
retval = []
for i in range(crops_multiplier):
retval.append("WHEAT" + str(i))
retval.append("CORN" + str(i))
retval.append("SUGAR_BEETS" + str(i))
return retval
model.CROPS = pyo.Set(initialize=crops_init)
#
# Parameters
#
model.TOTAL_ACREAGE = 500.0 * crops_multiplier
def _scale_up_data(indict):
outdict = {}
for i in range(crops_multiplier):
for crop in ['WHEAT', 'CORN', 'SUGAR_BEETS']:
outdict[crop + str(i)] = indict[crop]
return outdict
model.PriceQuota = _scale_up_data({
'WHEAT': 100000.0,
'CORN': 100000.0,
'SUGAR_BEETS': 6000.0
})
model.SubQuotaSellingPrice = _scale_up_data({
'WHEAT': 170.0,
'CORN': 150.0,
'SUGAR_BEETS': 36.0
})
model.SuperQuotaSellingPrice = _scale_up_data({
'WHEAT': 0.0,
'CORN': 0.0,
'SUGAR_BEETS': 10.0
})
model.CattleFeedRequirement = _scale_up_data({
'WHEAT': 200.0,
'CORN': 240.0,
'SUGAR_BEETS': 0.0
})
model.PurchasePrice = _scale_up_data({
'WHEAT': 238.0,
'CORN': 210.0,
'SUGAR_BEETS': 100000.0
})
model.PlantingCostPerAcre = _scale_up_data({
'WHEAT': 150.0,
'CORN': 230.0,
'SUGAR_BEETS': 260.0
})
#
# Stochastic Data
#
Yield = {}
Yield['BelowAverageScenario'] = \
{'WHEAT':2.0,'CORN':2.4,'SUGAR_BEETS':16.0}
Yield['AverageScenario'] = \
{'WHEAT':2.5,'CORN':3.0,'SUGAR_BEETS':20.0}
Yield['AboveAverageScenario'] = \
{'WHEAT':3.0,'CORN':3.6,'SUGAR_BEETS':24.0}
def Yield_init(m, cropname):
# yield as in "crop yield"
crop_base_name = cropname.rstrip("0123456789")
if scengroupnum != 0:
return Yield[scenario_base_name][
crop_base_name] + farmerstream.rand()
else:
return Yield[scenario_base_name][crop_base_name]
model.Yield = pyo.Param(model.CROPS,
within=pyo.NonNegativeReals,
initialize=Yield_init,
mutable=True)
#
# Variables
#
if (use_integer):
model.DevotedAcreage = pyo.Var(model.CROPS,
within=pyo.NonNegativeIntegers,
bounds=(0.0, model.TOTAL_ACREAGE))
else:
model.DevotedAcreage = pyo.Var(model.CROPS,
bounds=(0.0, model.TOTAL_ACREAGE))
model.QuantitySubQuotaSold = pyo.Var(model.CROPS, bounds=(0.0, None))
model.QuantitySuperQuotaSold = pyo.Var(model.CROPS, bounds=(0.0, None))
model.QuantityPurchased = pyo.Var(model.CROPS, bounds=(0.0, None))
#
# Constraints
#
def ConstrainTotalAcreage_rule(model):
return pyo.sum_product(model.DevotedAcreage) <= model.TOTAL_ACREAGE
model.ConstrainTotalAcreage = pyo.Constraint(
rule=ConstrainTotalAcreage_rule)
def EnforceCattleFeedRequirement_rule(model, i):
return model.CattleFeedRequirement[i] <= (
model.Yield[i] * model.DevotedAcreage[i]
) + model.QuantityPurchased[i] - model.QuantitySubQuotaSold[
i] - model.QuantitySuperQuotaSold[i]
model.EnforceCattleFeedRequirement = pyo.Constraint(
model.CROPS, rule=EnforceCattleFeedRequirement_rule)
def LimitAmountSold_rule(model, i):
return model.QuantitySubQuotaSold[i] + model.QuantitySuperQuotaSold[
i] - (model.Yield[i] * model.DevotedAcreage[i]) <= 0.0
model.LimitAmountSold = pyo.Constraint(model.CROPS,
rule=LimitAmountSold_rule)
def EnforceQuotas_rule(model, i):
return (0.0, model.QuantitySubQuotaSold[i], model.PriceQuota[i])
model.EnforceQuotas = pyo.Constraint(model.CROPS, rule=EnforceQuotas_rule)
# Stage-specific cost computations;
def ComputeFirstStageCost_rule(model):
return pyo.sum_product(model.PlantingCostPerAcre, model.DevotedAcreage)
model.FirstStageCost = pyo.Expression(rule=ComputeFirstStageCost_rule)
def ComputeSecondStageCost_rule(model):
expr = pyo.sum_product(model.PurchasePrice, model.QuantityPurchased)
expr -= pyo.sum_product(model.SubQuotaSellingPrice,
model.QuantitySubQuotaSold)
expr -= pyo.sum_product(model.SuperQuotaSellingPrice,
model.QuantitySuperQuotaSold)
return expr
model.SecondStageCost = pyo.Expression(rule=ComputeSecondStageCost_rule)
def total_cost_rule(model):
if (sense == pyo.minimize):
return model.FirstStageCost + model.SecondStageCost
return -model.FirstStageCost - model.SecondStageCost
model.Total_Cost_Objective = pyo.Objective(rule=total_cost_rule,
sense=sense)
return model
def pysp2_callback(scenario_name, node_names=None, cb_data=None):
"""
mpisppy signature for scenario creation.
Then find a starting solution for the scenario if solver option is not None.
Note that stage numbers are one-based.
Args:
scenario_name (str): put the scenario number on the end
node_names (int): not used
cb_data: (dict) "etree", "solver", "epath", "tee", "acstream",
"convex_relaxation"
Returns:
scenario (pyo.ConcreteModel): the scenario instance
Attaches:
_enodes (ACtree nodes): a list of the ACtree tree nodes
_egret_md (egret tuple with dict as [1]) egret model data
"""
# pull the number off the end of the scenario name
scen_num = sputils.extract_num(scenario_name)
etree = cb_data["etree"]
solver = cb_data["solver"]
acstream = cb_data["acstream"]
convex_relaxation = cb_data["convex_relaxation"] if "convex_relaxation"\
in cb_data else False
def lines_up_and_down(stage_md_dict, enode):
# local routine to configure the lines in stage_md_dict for the scenario
LinesDown = []
for f in enode.FailedLines:
LinesDown.append(f[0])
for this_branch in stage_md_dict.elements("branch"):
if this_branch[0] in enode.LinesUp:
this_branch[1]["in_service"] = True
elif this_branch[0] in LinesDown:
this_branch[1]["in_service"] = False
else:
print("enode.LinesUp=", enode.LinesUp)
print("enode.FailedLines=", enode.FailedLines)
raise RuntimeError("Branch (line) {} neither up nor down in scenario {}".\
format(this_branch[0], scenario_name))
def _egret_model(md_dict):
# the exact acopf model is hard-wired here:
if not convex_relaxation:
pyomod, mdict = eac.create_riv_acopf_model(
md_dict, include_feasibility_slack=True)
else:
pyomod, mdict = eac_relax.create_soc_relaxation(
md_dict,
include_feasibility_slack=True,
use_linear_relaxation=False)
return pyomod, mdict
# pull the number off the end of the scenario name
scen_num = sputils.extract_num(scenario_name)
#print ("debug scen_num=",scen_num)
numstages = etree.NumStages
enodes = etree.Nodes_for_Scenario(scen_num)
full_scenario_model = pyo.ConcreteModel()
full_scenario_model.stage_models = dict()
# look at egret/data/model_data.py for the format specification of md_dict
first_stage_md_dict = _md_dict(cb_data)
generator_set = first_stage_md_dict.attributes("generator")
generator_names = generator_set["names"]
# the following creates the first stage model
full_scenario_model.stage_models[1], model_dict = _egret_model(
first_stage_md_dict)
full_scenario_model.stage_models[1].obj.deactivate()
setattr(full_scenario_model, "stage_models_" + str(1),
full_scenario_model.stage_models[1])
for stage in range(2, numstages + 1):
#print ("stage={}".format(stage))
stage_md_dict = copy.deepcopy(first_stage_md_dict)
#print ("debug: processing node {}".format(enodes[stage-1].Name))
lines_up_and_down(stage_md_dict, enodes[stage - 1])
full_scenario_model.stage_models[stage], model_dict = \
_egret_model(stage_md_dict)
full_scenario_model.stage_models[stage].obj.deactivate()
setattr(full_scenario_model, "stage_models_" + str(stage),
full_scenario_model.stage_models[stage])
def aggregate_ramping_rule(m):
"""
We are adding ramping to the obj instead of a constraint for now
because we may not have ramp limit data.
"""
retval = 0
for stage in range(1, numstages):
retval += sum((full_scenario_model.stage_models[stage+1].pg[this_gen]\
- full_scenario_model.stage_models[stage].pg[this_gen])**2\
for this_gen in generator_names)
return retval
full_scenario_model.ramping = pyo.Expression(rule=aggregate_ramping_rule)
full_scenario_model.objective = pyo.Objective(expr=\
1000000.0 * full_scenario_model.ramping+\
sum(full_scenario_model.stage_models[stage].obj.expr\
for stage in range(1,numstages+1)))
inst = full_scenario_model
# end code from PySP1
node_list = list()
parent_name = None
for sm1, enode in enumerate(etree.Nodes_for_Scenario(scen_num)):
stage = sm1 + 1
if stage < etree.NumStages:
node_list.append(
scenario_tree.ScenarioNode(
name=enode.Name,
cond_prob=enode.CondProb,
stage=stage,
cost_expression=inst.stage_models[stage].obj,
scen_name_list=enode.ScenarioList,
nonant_list=[
inst.stage_models[stage].pg,
inst.stage_models[stage].qg
],
scen_model=inst,
parent_name=parent_name))
parent_name = enode.Name
inst._PySPnode_list = node_list
# Optionally assign probability to PySP_prob
inst.PySP_prob = 1 / etree.numscens
# solve it so subsequent code will have a good start
if solver is not None:
print(
f"scenario creation callback is solving {scenario_name} on rank {rank}"
)
solver.solve(
inst) #, tee=True) #symbolic_solver_labels=True, keepfiles=True)
# attachments
inst._enodes = enodes
inst._egret_md = first_stage_md_dict
return inst
def iterk(self, PHIter):
""" Before iter k>1 solves, but after x-bar update.
"""
# first, do some persistent solver with bundles gymnastics
have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles
if have_bundles:
subpname = next(iter(self.ph.local_subproblems))
subp = self.ph.local_subproblems[subpname]
solver_is_persistent = isinstance(
subp._solver_plugin, pyo.pyomo.solvers.plugins.solvers.
persistent_solver.PersistentSolver)
if solver_is_persistent:
vars_to_update = {}
fixoptions = self.fixeroptions
raw_fixed_so_far = 0
self._update_fix_counts()
for sname, s in self.local_scenarios.items():
if self.fixer_tuples[s] is None:
print("MAJOR WARNING: No Iter k fixer tuple for s.name=",
s.name)
return
if not have_bundles:
solver_is_persistent = isinstance(
s._solver_plugin, pyo.pyomo.solvers.plugins.solvers.
persistent_solver.PersistentSolver)
for (varid, th, nb, lb, ub) in self.fixer_tuples[s]:
was_fixed = False
try:
(ndn, i) = s._varid_to_nonant_index[varid]
except:
print("Are you trying to fix a Var that is not nonant?")
raise
tolval = self.threshold[(ndn, i)]
xvar = s._nonant_indexes[ndn, i]
if not xvar.is_fixed():
xb = pyo.value(s._xbars[(ndn, i)])
fx = s._PySP_conv_iter_count[(ndn, i)]
if fx > 0:
xbar = pyo.value(s._xbars[(ndn, i)])
was_fixed = False
if nb is not None and nb <= fx:
xvar.fix(xbar)
self._vb("Fixed nb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif lb is not None and lb < fx \
and xb - xvar.lb < self.boundtol:
xvar.fix(xvar.lb)
self._vb("Fixed lb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif ub is not None and ub < fx \
and xvar.ub - xb < self.boundtol:
xvar.fix(xvar.ub)
self._vb("Fixed ub %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
if was_fixed:
raw_fixed_so_far += 1
if not have_bundles and solver_is_persistent:
s._solver_plugin.update_var(xvar)
if have_bundles and solver_is_persistent:
if sname not in vars_to_update:
vars_to_update[sname] = []
vars_to_update[sname].append(xvar)
if have_bundles and solver_is_persistent:
for k, subp in self.ph.local_subproblems.items():
subpnum = sputils.extract_num(k)
rank_local = self.ph.rank
for sname in self.ph.names_in_bundles[rank_local][subpnum]:
if sname in vars_to_update:
for xvar in vars_to_update[sname]:
subp._solver_plugin.update_var(xvar)
self.fixed_so_far += raw_fixed_so_far / len(self.local_scenarios)
self._dp("Unique Vars fixed so far %s" % (self.fixed_so_far))
if raw_fixed_so_far % len(self.local_scenarios) != 0:
raise RuntimeError("Variation in fixing across scenarios detected "
"in fixer.py")
def scenario_creator(
scenario_name,
use_integer=False,
sense=pyo.minimize,
crops_multiplier=1,
):
""" Create a scenario for the (scalable) farmer example.
Args:
scenario_name (str):
Name of the scenario to construct.
use_integer (bool, optional):
If True, restricts variables to be integer. Default is False.
sense (int, optional):
Model sense (minimization or maximization). Must be either
pyo.minimize or pyo.maximize. Default is pyo.minimize.
crops_multiplier (int, optional):
Factor to control scaling. There will be three times this many
crops. Default is 1.
"""
# scenario_name has the form <str><int> e.g. scen12, foobar7
# The digits are scraped off the right of scenario_name using regex then
# converted mod 3 into one of the below avg./avg./above avg. scenarios
scennum = sputils.extract_num(scenario_name)
basenames = [
'BelowAverageScenario', 'AverageScenario', 'AboveAverageScenario'
]
basenum = scennum % 3
groupnum = scennum // 3
scenname = basenames[basenum] + str(groupnum)
# The RNG is seeded with the scenario number so that it is
# reproducible when used with multiple threads.
# NOTE: if you want to do replicates, you will need to pass a seed
# as a kwarg to scenario_creator then use seed+scennum as the seed argument.
farmerstream.seed(scennum)
# Check for minimization vs. maximization
if sense not in [pyo.minimize, pyo.maximize]:
raise ValueError("Model sense Not recognized")
# Create the concrete model object
model = pysp_instance_creation_callback(
scenname,
use_integer=use_integer,
sense=sense,
crops_multiplier=crops_multiplier,
)
# Create the list of nodes associated with the scenario (for two stage,
# there is only one node associated with the scenario--leaf nodes are
# ignored).
model._PySPnode_list = [
scenario_tree.ScenarioNode(
name="ROOT",
cond_prob=1.0,
stage=1,
cost_expression=model.FirstStageCost,
scen_name_list=None, # Deprecated?
nonant_list=[model.DevotedAcreage],
scen_model=model,
)
]
return model
def iter0(self, local_scenarios):
# first, do some persistent solver with bundles gymnastics
have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles
if have_bundles:
subpname = next(iter(self.ph.local_subproblems))
subp = self.ph.local_subproblems[subpname]
solver_is_persistent = isinstance(
subp._solver_plugin, pyo.pyomo.solvers.plugins.solvers.
persistent_solver.PersistentSolver)
if solver_is_persistent:
vars_to_update = {}
fixoptions = self.fixeroptions
raw_fixed_so_far = 0 # count those fixed in each scenario
for sname, s in self.ph.local_scenarios.items():
if self.iter0_fixer_tuples[s] is None:
print("WARNING: No Iter0 fixer tuple for s.name=", s.name)
return
if not have_bundles:
solver_is_persistent = isinstance(
s._solver_plugin, pyo.pyomo.solvers.plugins.solvers.
persistent_solver.PersistentSolver)
for (varid, th, nb, lb, ub) in self.iter0_fixer_tuples[s]:
was_fixed = False
try:
(ndn, i) = s._varid_to_nonant_index[varid]
except:
print("Are you trying to fix a Var that is not nonant?")
raise
xvar = s._nonant_indexes[ndn, i]
if not xvar.is_fixed():
xb = pyo.value(s._xbars[(ndn, i)])
diff = xb * xb - pyo.value(s._xsqbars[(ndn, i)])
tolval = self.iter0_threshold[(ndn, i)]
sqtolval = tolval * tolval # the tol is on sqrt
if -diff > sqtolval or diff > sqtolval:
##print ("debug0 NO fix diff, sqtolval", diff, sqtolval)
continue
else:
##print ("debug0 fix diff, sqtolval", diff, sqtolval)
# if we are still here, it is converged
if nb is not None:
xvar.fix(xb)
self._vb("Fixed0 nb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif lb is not None and xb - xvar.lb < self.boundtol:
xvar.fix(xvar.lb)
self._vb("Fixed0 lb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif ub is not None and xvar.ub - xb < self.boundtol:
xvar.fix(xvar.ub)
self._vb("Fixed0 ub %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
if was_fixed:
raw_fixed_so_far += 1
if not have_bundles and solver_is_persistent:
s._solver_plugin.update_var(xvar)
if have_bundles and solver_is_persistent:
if sname not in vars_to_update:
vars_to_update[sname] = []
vars_to_update[sname].append(xvar)
if have_bundles and solver_is_persistent:
for k, subp in self.ph.local_subproblems.items():
subpnum = sputils.extract_num(k)
rank_local = self.ph.rank
for sname in self.ph.names_in_bundles[rank_local][subpnum]:
if sname in vars_to_update:
for xvar in vars_to_update[sname]:
subp._solver_plugin.update_var(xvar)
self.fixed_so_far += raw_fixed_so_far / len(local_scenarios)
self._dp("Unique Vars fixed so far %s" % (self.fixed_so_far))
if raw_fixed_so_far % len(local_scenarios) != 0:
raise RuntimeError("Variation in fixing across scenarios detected "
"in fixer.py (iter0)")
except:
print(msg, "\n bad number of bundles per rank=", sys.argv[2])
quit()
try:
maxit = int(sys.argv[3])
except:
print(msg, "\n bad max iterations=", sys.argv[3])
quit()
try:
rho = int(sys.argv[4])
except:
print(msg, "\n bad rho=", sys.argv[4])
quit()
# The number of scenarios is the last number in the instance name
ScenCount = sputils.extract_num(instname)
start_time = dt.datetime.now()
PHoptions = {}
PHoptions["solvername"] = "gurobi_persistent"
PHoptions["PHIterLimit"] = maxit
PHoptions["defaultPHrho"] = rho
PHoptions["convthresh"] = -1
PHoptions["subsolvedirectives"] = None
PHoptions["verbose"] = False
PHoptions["display_timing"] = False
PHoptions["display_progress"] = True
### async section ###
PHoptions["asynchronous"] = True
PHoptions["async_frac_needed"] = 0.5
def pysp2_callback(
scenario_name,
etree=None,
solver=None,
epath=None,
tee=False,
acstream=None,
convex_relaxation=False,
verbose=False,
ramp_coeff=1000000,
load_mismatch_cost=1000,
q_load_mismatch_cost=None,
):
"""
mpisppy signature for scenario creation.
Then find a starting solution for the scenario if solver option is not None.
Note that stage numbers are one-based.
Args:
scenario_name (str):
Put the scenario number on the end
etree ():
Default is None.
solver (str):
Solver to use.
epath (str):
Path to the egret data
tee (bool):
If True, displays solver output. Default is False.
acstream ():
Default is None.
convex_relaxation (bool):
If True, build the convex relaxation. Default is False.
verbose (bool, optional):
If True, display verbose output. Default is False.
ramp_coeff (int, optional):
Default is 1e6.
load_mismatch_cost (float, optional):
Default is 1000.
q_load_mismatch_cost (float, optional):
Deafult is load_mismatch_cost.
Returns:
scenario (pyo.ConcreteModel): the scenario instance
Attaches:
_enodes (ACtree nodes): a list of the ACtree tree nodes
_egret_md (egret tuple with dict as [1]) egret model data
"""
print("Debug: convex_relaxation=", convex_relaxation)
# pull the number off the end of the scenario name
scen_num = sputils.extract_num(scenario_name)
if q_load_mismatch_cost is None:
q_load_mismatch_cost = load_mismatch_cost
def lines_up_and_down(stage_md_dict, enode):
# local routine to configure the lines in stage_md_dict for the scenario
LinesDown = []
for f in enode.FailedLines:
LinesDown.append(f[0])
for this_branch in stage_md_dict.elements("branch"):
if this_branch[0] in enode.LinesUp:
this_branch[1]["in_service"] = True
elif this_branch[0] in LinesDown:
this_branch[1]["in_service"] = False
else:
print("enode.LinesUp=", enode.LinesUp)
print("enode.FailedLines=", enode.FailedLines)
raise RuntimeError("Branch (line) {} neither up nor down in scenario {}".\
format(this_branch[0], scenario_name))
def _egret_model(md_dict):
# the exact acopf model is hard-wired here:
md_dict.data['system']['load_mismatch_cost'] = load_mismatch_cost
md_dict.data['system']['q_load_mismatch_cost'] = q_load_mismatch_cost
if not convex_relaxation:
pyomod, mdict = eac.create_riv_acopf_model(
md_dict, include_feasibility_slack=True)
else:
pyomod, mdict = eac_relax.create_soc_relaxation(
md_dict,
include_feasibility_slack=True,
use_linear_relaxation=False)
return pyomod, mdict
# pull the number off the end of the scenario name
scen_num = sputils.extract_num(scenario_name)
numstages = etree.NumStages
enodes = etree.Nodes_for_Scenario(scen_num)
full_scenario_model = pyo.ConcreteModel()
full_scenario_model.stage_models = dict()
# look at egret/data/model_data.py for the format specification of md_dict
first_stage_md_dict = _md_dict(epath)
generator_set = first_stage_md_dict.attributes("generator")
generator_names = generator_set["names"]
# the following creates the first stage model
full_scenario_model.stage_models[1], model_dict = _egret_model(
first_stage_md_dict)
full_scenario_model.stage_models[1].obj.deactivate()
setattr(full_scenario_model, "stage_models_" + str(1),
full_scenario_model.stage_models[1])
for stage in range(2, numstages + 1):
#print ("stage={}".format(stage))
stage_md_dict = copy.deepcopy(first_stage_md_dict)
#print ("debug: processing node {}".format(enodes[stage-1].Name))
lines_up_and_down(stage_md_dict, enodes[stage - 1])
full_scenario_model.stage_models[stage], model_dict = \
_egret_model(stage_md_dict)
full_scenario_model.stage_models[stage].obj.deactivate()
setattr(full_scenario_model, "stage_models_" + str(stage),
full_scenario_model.stage_models[stage])
def aggregate_ramping_rule(m):
"""
We are adding ramping to the obj instead of a constraint for now
because we may not have ramp limit data.
"""
retval = 0
for stage in range(1, numstages):
retval += sum((full_scenario_model.stage_models[stage+1].pg[this_gen]\
- full_scenario_model.stage_models[stage].pg[this_gen])**2\
for this_gen in generator_names)
return retval
full_scenario_model.ramping = pyo.Expression(rule=aggregate_ramping_rule)
full_scenario_model.objective = pyo.Objective(expr=\
ramp_coeff * full_scenario_model.ramping+\
sum(full_scenario_model.stage_models[stage].obj.expr\
for stage in range(1,numstages+1)))
inst = full_scenario_model
# end code from PySP1
node_list = list()
parent_name = None
for sm1, enode in enumerate(etree.Nodes_for_Scenario(scen_num)):
stage = sm1 + 1
if stage < etree.NumStages:
node_list.append(
scenario_tree.ScenarioNode(
name=enode.Name,
cond_prob=enode.CondProb,
stage=stage,
cost_expression=inst.stage_models[stage].obj,
scen_name_list=enode.ScenarioList,
nonant_list=[
inst.stage_models[stage].pg,
inst.stage_models[stage].qg
],
scen_model=inst,
parent_name=parent_name))
parent_name = enode.Name
if verbose:
print(f"\nScenario creation for {scenario_name} on rank {rank}"
f" FailedLines (line,minutes)={enode.FailedLines}")
inst._PySPnode_list = node_list
inst.PySP_prob = 1 / etree.numscens
# solve it so subsequent code will have a good start
if solver is not None:
print(
f"scenario creation callback is solving {scenario_name} on rank {rank}"
)
solver.solve(inst,
tee=tee) #symbolic_solver_labels=True, keepfiles=True)
# attachments
inst._enodes = enodes
inst._egret_md = first_stage_md_dict
return inst
def gap_estimators(xhat_one,
mname,
solving_type="EF-2stage",
scenario_names=None,
sample_options=None,
ArRP=1,
scenario_creator_kwargs={},
scenario_denouement=None,
solvername=None,
solver_options=None,
verbose=False,
objective_gap=False):
''' Given a xhat, scenario names, a scenario creator and options,
gap_estimators creates a scenario tree and the associatd estimators
G and s from §2 of [bm2011].
Returns G and s evaluated at xhat.
If ArRP>1, G and s are pooled, from a number ArRP of estimators,
computed with different scenario trees.
Parameters
----------
xhat_one : dict
A candidate first stage solution
mname: str
Name of the reference model, e.g. 'mpisppy.tests.examples.farmer'.
solving_type: str, optional
The way we solve the approximate problem. Can be "EF-2stage" (default)
or "EF-mstage".
scenario_names: list, optional
List of scenario names used to compute G_n and s_n. Default is None
Must be specified for 2 stage, but can be missing for multistage
sample_options: dict, optional
Only for multistage. Must contain a 'seed' and a 'branching_factors' attribute,
specifying the starting seed and the branching factors
of the scenario tree
ArRP:int,optional
Number of batches (we create a ArRP model). Default is 1 (one batch).
scenario_creator_kwargs: dict, optional
Additional arguments for scenario_creator. Default is {}
scenario_denouement: function, optional
Function to run after scenario creation. Default is None.
solvername : str, optional
Solver. Default is None
solver_options: dict, optional
Solving options. Default is None
verbose: bool, optional
Should it print the gap estimator ? Default is True
objective_gap: bool, optional
Returns a gap estimate around approximate objective value
branching_factors: list, optional
Only for multistage. List of branching factors of the sample scenario tree.
Returns
-------
G_k and s_k, gap estimator and associated standard deviation estimator.
'''
global_toc("Enter gap_estimators")
if solving_type not in ["EF-2stage", "EF-mstage"]:
raise RuntimeError(
"Only EF solve for the approximate problem is supported yet.")
else:
is_multi = (solving_type == "EF-mstage")
if is_multi:
try:
branching_factors = sample_options['branching_factors']
start = sample_options['seed']
except (TypeError, KeyError, RuntimeError):
raise RuntimeError(
'For multistage problems, sample_options must be a dict with branching_factors and seed attributes.'
)
else:
start = sputils.extract_num(scenario_names[0])
if ArRP > 1: #Special case : ArRP, G and s are pooled from r>1 estimators.
if is_multi:
raise RuntimeError(
"Pooled estimators are not supported for multistage problems yet."
)
n = len(scenario_names)
if (n % ArRP != 0):
raise RuntimeWarning("You put as an input a number of scenarios"+\
f" which is not a mutliple of {ArRP}.")
n = n - n % ArRP
G = []
s = []
for k in range(ArRP):
scennames = scenario_names[k * (n // ArRP):(k + 1) * (n // ArRP)]
tmp = gap_estimators(
xhat_one,
mname,
solvername=solvername,
scenario_names=scennames,
ArRP=1,
scenario_creator_kwargs=scenario_creator_kwargs,
scenario_denouement=scenario_denouement,
solver_options=solver_options,
solving_type=solving_type)
G.append(tmp['G'])
s.append(tmp['s'])
global_toc(f"ArRP {k} of {ArRP}")
#Pooling
G = np.mean(G)
s = np.linalg.norm(s) / np.sqrt(n // ArRP)
return {"G": G, "s": s, "seed": start}
#A1RP
#We start by computing the optimal solution to the approximate problem induced by our scenarios
if is_multi:
#Sample a scenario tree: this is a subtree, but starting from stage 1
samp_tree = sample_tree.SampleSubtree(
mname,
xhats=[],
root_scen=None,
starting_stage=1,
branching_factors=branching_factors,
seed=start,
options=scenario_creator_kwargs,
solvername=solvername,
solver_options=solver_options)
samp_tree.run()
start += sputils.number_of_nodes(branching_factors)
ama_object = samp_tree.ama
else:
#We use amalgamator to do it
ama_options = dict(scenario_creator_kwargs)
ama_options['start'] = start
ama_options['num_scens'] = len(scenario_names)
ama_options['EF_solver_name'] = solvername
ama_options['EF_solver_options'] = solver_options
ama_options[solving_type] = True
ama_object = ama.from_module(mname,
ama_options,
use_command_line=False)
ama_object.scenario_names = scenario_names
ama_object.verbose = False
ama_object.run()
start += len(scenario_names)
#Optimal solution of the approximate problem
zstar = ama_object.best_outer_bound
#Associated policies
xstars = sputils.nonant_cache_from_ef(ama_object.ef)
#Then, we evaluate the fonction value induced by the scenario at xstar.
if is_multi:
# Find feasible policies (i.e. xhats) for every non-leaf nodes
if len(samp_tree.ef._ef_scenario_names) > 1:
local_scenarios = {
sname: getattr(samp_tree.ef, sname)
for sname in samp_tree.ef._ef_scenario_names
}
else:
local_scenarios = {
samp_tree.ef._ef_scenario_names[0]: samp_tree.ef
}
xhats, start = sample_tree.walking_tree_xhats(
mname,
local_scenarios,
xhat_one['ROOT'],
branching_factors,
start,
scenario_creator_kwargs,
solvername=solvername,
solver_options=solver_options)
#Compute then the average function value with this policy
scenario_creator_kwargs = samp_tree.ama.kwargs
all_nodenames = sputils.create_nodenames_from_branching_factors(
branching_factors)
else:
#In a 2 stage problem, the only non-leaf is the ROOT node
xhats = xhat_one
all_nodenames = None
xhat_eval_options = {
"iter0_solver_options": None,
"iterk_solver_options": None,
"display_timing": False,
"solvername": solvername,
"verbose": False,
"solver_options": solver_options
}
ev = xhat_eval.Xhat_Eval(xhat_eval_options,
scenario_names,
ama_object.scenario_creator,
scenario_denouement,
scenario_creator_kwargs=scenario_creator_kwargs,
all_nodenames=all_nodenames)
#Evaluating xhat and xstar and getting the value of the objective function
#for every (local) scenario
zhat = ev.evaluate(xhats)
objs_at_xhat = ev.objs_dict
zstar = ev.evaluate(xstars)
objs_at_xstar = ev.objs_dict
eval_scen_at_xhat = []
eval_scen_at_xstar = []
scen_probs = []
for k, s in ev.local_scenarios.items():
eval_scen_at_xhat.append(objs_at_xhat[k])
eval_scen_at_xstar.append(objs_at_xstar[k])
scen_probs.append(s._mpisppy_probability)
scen_gaps = np.array(eval_scen_at_xhat) - np.array(eval_scen_at_xstar)
local_gap = np.dot(scen_gaps, scen_probs)
local_ssq = np.dot(scen_gaps**2, scen_probs)
local_prob_sqnorm = np.linalg.norm(scen_probs)**2
local_obj_at_xhat = np.dot(eval_scen_at_xhat, scen_probs)
local_estim = np.array(
[local_gap, local_ssq, local_prob_sqnorm, local_obj_at_xhat])
global_estim = np.zeros(4)
ev.mpicomm.Allreduce(local_estim, global_estim, op=mpi.SUM)
G, ssq, prob_sqnorm, obj_at_xhat = global_estim
if global_rank == 0 and verbose:
print(f"G = {G}")
sample_var = (ssq - G**2) / (1 - prob_sqnorm) #Unbiased sample variance
s = np.sqrt(sample_var)
use_relative_error = (np.abs(zstar) > 1)
G = correcting_numeric(G,
objfct=obj_at_xhat,
relative_error=use_relative_error)
if objective_gap:
if is_multi:
return {
"G": G,
"s": s,
"zhats": [zhat],
"zstars": [zstar],
"seed": start
}
else:
return {
"G": G,
"s": s,
"zhats": eval_scen_at_xhat,
"zstars": eval_scen_at_xstar,
"seed": start
}
else:
return {"G": G, "s": s, "seed": start}
def iter0(self, local_scenarios):
# first, do some persistent solver with bundles gymnastics
have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles
if have_bundles:
subpname = next(iter(self.ph.local_subproblems))
subp = self.ph.local_subproblems[subpname]
solver_is_persistent = isinstance(subp._solver_plugin,
pyo.pyomo.solvers.plugins.solvers.persistent_solver.PersistentSolver)
if solver_is_persistent:
vars_to_update = {}
fixoptions = self.fixeroptions
# modelers might have already fixed variables - count those up and output the result
raw_fixed_on_arrival = 0
raw_fixed_this_iter = 0
for sname,s in self.ph.local_scenarios.items():
if self.iter0_fixer_tuples[s] is None:
print ("WARNING: No Iter0 fixer tuple for s.name=",s.name)
return
if not have_bundles:
solver_is_persistent = isinstance(s._solver_plugin,
pyo.pyomo.solvers.plugins.solvers.persistent_solver.PersistentSolver)
for (varid, th, nb, lb, ub) in self.iter0_fixer_tuples[s]:
was_fixed = False
try:
(ndn, i) = s._mpisppy_data.varid_to_nonant_index[varid]
except:
print ("Are you trying to fix a Var that is not nonant?")
raise
xvar = s._mpisppy_data.nonant_indices[ndn,i]
if not xvar.is_fixed():
xb = pyo.value(s._mpisppy_model.xbars[(ndn,i)])
diff = xb * xb - pyo.value(s._mpisppy_model.xsqbars[(ndn,i)])
tolval = self.iter0_threshold[(ndn, i)]
sqtolval = tolval*tolval # the tol is on sqrt
if -diff > sqtolval or diff > sqtolval:
##print ("debug0 NO fix diff, sqtolval", diff, sqtolval)
continue
else:
##print ("debug0 fix diff, sqtolval", diff, sqtolval)
# if we are still here, it is converged
if nb is not None:
xvar.fix(xb)
self._vb("Fixed0 nb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif lb is not None and xb - xvar.lb < self.boundtol:
xvar.fix(xvar.lb)
self._vb("Fixed0 lb %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
elif ub is not None and xvar.ub - xb < self.boundtol:
xvar.fix(xvar.ub)
self._vb("Fixed0 ub %s %s at %s" % \
(s.name, xvar.name, str(xvar._value)))
was_fixed = True
if was_fixed:
raw_fixed_this_iter += 1
if not have_bundles and solver_is_persistent:
s._solver_plugin.update_var(xvar)
if have_bundles and solver_is_persistent:
if sname not in vars_to_update:
vars_to_update[sname] = []
vars_to_update[sname].append(xvar)
else:
# TODO: a paranoid would and should put a check to ensure
# that variables are fixed in all scenarios and are fixed
# to the same value.
raw_fixed_on_arrival += 1
if have_bundles and solver_is_persistent:
for k,subp in self.ph.local_subproblems.items():
subpnum = sputils.extract_num(k)
rank_local = self.ph.cylinder_rank
for sname in self.ph.names_in_bundles[rank_local][subpnum]:
if sname in vars_to_update:
for xvar in vars_to_update[sname]:
subp._solver_plugin.update_var(xvar)
self.fixed_prior_iter0 += raw_fixed_on_arrival / len(local_scenarios)
self.fixed_so_far += raw_fixed_this_iter / len(local_scenarios)
self._dp("Unique vars fixed so far - %d (%d prior to iteration 0)" % (self.fixed_so_far+self.fixed_prior_iter0, self.fixed_prior_iter0))
if raw_fixed_this_iter % len(local_scenarios) != 0:
raise RuntimeError ("Variation in fixing across scenarios detected "
"in fixer.py (iter0)")
# maybe to do mpicomm.abort()
if raw_fixed_on_arrival % len(local_scenarios) != 0:
raise RuntimeError ("Variation in fixing across scenarios prior to iteration 0 detected "
"in fixer.py (iter0)")
