def feasible_solution(mname,
scenario,
xhat_one,
BFs,
seed,
options,
solvername="gurobi",
solver_options=None):
'''
Given a scenario and a first-stage policy xhat_one, this method computes
non-anticipative feasible policies for the following stages.
'''
if xhat_one is None:
raise RuntimeError("Xhat_one can't be None for now")
ciutils.is_sorted(scenario._mpisppy_node_list)
nodenames = [node.name for node in scenario._mpisppy_node_list]
num_stages = len(BFs) + 1
xhats = [xhat_one]
for t in range(2, num_stages): #We do not compute xhat for the final stage
subtree = SampleSubtree(mname, xhats, scenario, t, BFs, seed, options,
solvername, solver_options)
subtree.run()
xhats.append(subtree.xhat_at_stage)
seed += sputils.number_of_nodes(BFs[(t - 1):])
xhat_dict = {ndn: xhat for (ndn, xhat) in zip(nodenames, xhats)}
return xhat_dict, seed
def run(self,maxit=200):
refmodel = self.refmodel
mult = self.sample_size_ratio # used to set m_k= mult*n_k
#----------------------------Step 0 -------------------------------------#
#Initialization
k =1
#Computing the lower bound for n_1
if self.stopping_criterion == "BM":
#Finding a constant used to compute nk
r = 2 #TODO : we could add flexibility here
j = np.arange(1,1000)
if self.q is None:
s = sum(np.power(j,-self.p*np.log(j)))
else:
if self.q<1:
raise RuntimeError("Parameter q should be greater than 1.")
s = sum(np.exp(-self.p*np.power(j,2*self.q/r)))
self.c = max(1,2*np.log(s/(np.sqrt(2*np.pi)*(1-self.confidence_level))))
lower_bound_k = self.sample_size(k, None, None, None)
#Computing xhat_1.
#We use sample_size_ratio*n_k observations to compute xhat_k
if self.multistage:
xhat_BFs = ciutils.scalable_BFs(mult*lower_bound_k, self.options['BFs'])
mk = np.prod(xhat_BFs)
self.xhat_gen_options['start_seed'] = self.SeedCount #TODO: Maybe find a better way to manage seed
xhat_scenario_names = refmodel.scenario_names_creator(mk)
else:
mk = int(np.floor(mult*lower_bound_k))
xhat_scenario_names = refmodel.scenario_names_creator(mk, start=self.ScenCount)
self.ScenCount+=mk
xhat_k = self.xhat_generator(xhat_scenario_names,
solvername=self.solvername,
solver_options=self.solver_options,
**self.xhat_gen_options)
#----------------------------Step 1 -------------------------------------#
#Computing n_1 and associated scenario names
if self.multistage:
self.SeedCount += sputils.number_of_nodes(xhat_BFs)
gap_BFs = ciutils.scalable_BFs(lower_bound_k, self.options['BFs'])
nk = np.prod(gap_BFs)
estimator_scenario_names = refmodel.scenario_names_creator(nk)
sample_options = {'BFs':gap_BFs, 'seed':self.SeedCount}
else:
nk = self.ArRP *int(np.ceil(lower_bound_k/self.ArRP))
estimator_scenario_names = refmodel.scenario_names_creator(nk,
start=self.ScenCount)
sample_options = None
self.ScenCount+= nk
#Computing G_nkand s_k associated with xhat_1
self.options['num_scens'] = nk
scenario_creator_kwargs = self.refmodel.kw_creator(self.options)
scenario_denouement = refmodel.scenario_denouement if hasattr(refmodel, "scenario_denouement") else None
estim = ciutils.gap_estimators(xhat_k, self.refmodelname,
solving_type=self.solving_type,
scenario_names=estimator_scenario_names,
sample_options=sample_options,
ArRP=self.ArRP,
scenario_creator_kwargs=scenario_creator_kwargs,
scenario_denouement=scenario_denouement,
solvername=self.solvername,
solver_options=self.solver_options)
Gk,sk = estim['G'],estim['s']
if self.multistage:
self.SeedCount = estim['seed']
#----------------------------Step 2 -------------------------------------#
while( self.stop_criterion(Gk,sk,nk) and k<maxit):
#----------------------------Step 3 -------------------------------------#
k+=1
nk_m1 = nk #n_{k-1}
mk_m1 = mk
lower_bound_k = self.sample_size(k, Gk, sk, nk_m1)
#Computing m_k and associated scenario names
if self.multistage:
xhat_BFs = ciutils.scalable_BFs(mult*lower_bound_k, self.options['BFs'])
mk = np.prod(xhat_BFs)
self.xhat_gen_options['start_seed'] = self.SeedCount #TODO: Maybe find a better way to manage seed
xhat_scenario_names = refmodel.scenario_names_creator(mk)
else:
mk = int(np.floor(mult*lower_bound_k))
assert mk>= mk_m1, "Our sample size should be increasing"
if (k%self.kf_xhat==0):
#We use only new scenarios to compute xhat
xhat_scenario_names = refmodel.scenario_names_creator(mult*nk,
start=self.ScenCount)
self.ScenCount+= mk
else:
#We reuse the previous scenarios
xhat_scenario_names+= refmodel.scenario_names_creator(mult*(nk-nk_m1),
start=self.ScenCount)
self.ScenCount+= mk-mk_m1
#Computing xhat_k
xhat_k = self.xhat_generator(xhat_scenario_names,
solvername=self.solvername,
solver_options=self.solver_options,
**self.xhat_gen_options)
#Computing n_k and associated scenario names
if self.multistage:
self.SeedCount += sputils.number_of_nodes(xhat_BFs)
gap_BFs = ciutils.scalable_BFs(lower_bound_k, self.options['BFs'])
nk = np.prod(gap_BFs)
estimator_scenario_names = refmodel.scenario_names_creator(nk)
sample_options = {'BFs':gap_BFs, 'seed':self.SeedCount}
else:
nk = self.ArRP *int(np.ceil(lower_bound_k/self.ArRP))
assert nk>= nk_m1, "Our sample size should be increasing"
if (k%self.kf_Gs==0):
#We use only new scenarios to compute gap estimators
estimator_scenario_names = refmodel.scenario_names_creator(nk,
start=self.ScenCount)
self.ScenCount+=nk
else:
#We reuse the previous scenarios
estimator_scenario_names+= refmodel.scenario_names_creator((nk-nk_m1),
start=self.ScenCount)
self.ScenCount+= (nk-nk_m1)
sample_options = None
#Computing G_k and s_k
self.options['num_scens'] = nk
scenario_creator_kwargs = self.refmodel.kw_creator(self.options)
estim = ciutils.gap_estimators(xhat_k, self.refmodelname,
solving_type=self.solving_type,
scenario_names=estimator_scenario_names,
sample_options=sample_options,
ArRP=self.ArRP,
scenario_creator_kwargs=scenario_creator_kwargs,
scenario_denouement=scenario_denouement,
solvername=self.solvername,
solver_options=self.solver_options)
if self.multistage:
self.SeedCount = estim['seed']
Gk,sk = estim['G'],estim['s']
if (k%10==0) and global_rank==0:
print(f"k={k}")
print(f"n_k={nk}")
print(f"G_k={Gk}")
print(f"s_k={sk}")
#----------------------------Step 4 -------------------------------------#
if (k==maxit) :
raise RuntimeError(f"The loop terminated after {maxit} iteration with no acceptable solution")
T = k
final_xhat=xhat_k
if self.stopping_criterion == "BM":
upper_bound=self.h*sk+self.eps
elif self.stopping_criterion == "BPL":
upper_bound = self.eps
else:
raise RuntimeError("Only BM and BPL criterion are supported yet.")
CI=[0,upper_bound]
global_toc(f"G={Gk}")
global_toc(f"s={sk}")
global_toc(f"xhat has been computed with {nk*mult} observations.")
return {"T":T,"Candidate_solution":final_xhat,"CI":CI,}
def gap_estimators(xhat_one,
mname,
solving_type="EF-2stage",
scenario_names=None,
sample_options=None,
ArRP=1,
scenario_creator_kwargs={},
scenario_denouement=None,
solvername=None,
solver_options=None,
verbose=False,
objective_gap=False):
''' Given a xhat, scenario names, a scenario creator and options,
gap_estimators creates a scenario tree and the associatd estimators
G and s from §2 of [bm2011].
Returns G and s evaluated at xhat.
If ArRP>1, G and s are pooled, from a number ArRP of estimators,
computed with different scenario trees.
Parameters
----------
xhat_one : dict
A candidate first stage solution
mname: str
Name of the reference model, e.g. 'mpisppy.tests.examples.farmer'.
solving_type: str, optional
The way we solve the approximate problem. Can be "EF-2stage" (default)
or "EF-mstage".
scenario_names: list, optional
List of scenario names used to compute G_n and s_n. Default is None
Must be specified for 2 stage, but can be missing for multistage
sample_options: dict, optional
Only for multistage. Must contain a 'seed' and a 'branching_factors' attribute,
specifying the starting seed and the branching factors
of the scenario tree
ArRP:int,optional
Number of batches (we create a ArRP model). Default is 1 (one batch).
scenario_creator_kwargs: dict, optional
Additional arguments for scenario_creator. Default is {}
scenario_denouement: function, optional
Function to run after scenario creation. Default is None.
solvername : str, optional
Solver. Default is None
solver_options: dict, optional
Solving options. Default is None
verbose: bool, optional
Should it print the gap estimator ? Default is True
objective_gap: bool, optional
Returns a gap estimate around approximate objective value
branching_factors: list, optional
Only for multistage. List of branching factors of the sample scenario tree.
Returns
-------
G_k and s_k, gap estimator and associated standard deviation estimator.
'''
global_toc("Enter gap_estimators")
if solving_type not in ["EF-2stage", "EF-mstage"]:
raise RuntimeError(
"Only EF solve for the approximate problem is supported yet.")
else:
is_multi = (solving_type == "EF-mstage")
if is_multi:
try:
branching_factors = sample_options['branching_factors']
start = sample_options['seed']
except (TypeError, KeyError, RuntimeError):
raise RuntimeError(
'For multistage problems, sample_options must be a dict with branching_factors and seed attributes.'
)
else:
start = sputils.extract_num(scenario_names[0])
if ArRP > 1: #Special case : ArRP, G and s are pooled from r>1 estimators.
if is_multi:
raise RuntimeError(
"Pooled estimators are not supported for multistage problems yet."
)
n = len(scenario_names)
if (n % ArRP != 0):
raise RuntimeWarning("You put as an input a number of scenarios"+\
f" which is not a mutliple of {ArRP}.")
n = n - n % ArRP
G = []
s = []
for k in range(ArRP):
scennames = scenario_names[k * (n // ArRP):(k + 1) * (n // ArRP)]
tmp = gap_estimators(
xhat_one,
mname,
solvername=solvername,
scenario_names=scennames,
ArRP=1,
scenario_creator_kwargs=scenario_creator_kwargs,
scenario_denouement=scenario_denouement,
solver_options=solver_options,
solving_type=solving_type)
G.append(tmp['G'])
s.append(tmp['s'])
global_toc(f"ArRP {k} of {ArRP}")
#Pooling
G = np.mean(G)
s = np.linalg.norm(s) / np.sqrt(n // ArRP)
return {"G": G, "s": s, "seed": start}
#A1RP
#We start by computing the optimal solution to the approximate problem induced by our scenarios
if is_multi:
#Sample a scenario tree: this is a subtree, but starting from stage 1
samp_tree = sample_tree.SampleSubtree(
mname,
xhats=[],
root_scen=None,
starting_stage=1,
branching_factors=branching_factors,
seed=start,
options=scenario_creator_kwargs,
solvername=solvername,
solver_options=solver_options)
samp_tree.run()
start += sputils.number_of_nodes(branching_factors)
ama_object = samp_tree.ama
else:
#We use amalgamator to do it
ama_options = dict(scenario_creator_kwargs)
ama_options['start'] = start
ama_options['num_scens'] = len(scenario_names)
ama_options['EF_solver_name'] = solvername
ama_options['EF_solver_options'] = solver_options
ama_options[solving_type] = True
ama_object = ama.from_module(mname,
ama_options,
use_command_line=False)
ama_object.scenario_names = scenario_names
ama_object.verbose = False
ama_object.run()
start += len(scenario_names)
#Optimal solution of the approximate problem
zstar = ama_object.best_outer_bound
#Associated policies
xstars = sputils.nonant_cache_from_ef(ama_object.ef)
#Then, we evaluate the fonction value induced by the scenario at xstar.
if is_multi:
# Find feasible policies (i.e. xhats) for every non-leaf nodes
if len(samp_tree.ef._ef_scenario_names) > 1:
local_scenarios = {
sname: getattr(samp_tree.ef, sname)
for sname in samp_tree.ef._ef_scenario_names
}
else:
local_scenarios = {
samp_tree.ef._ef_scenario_names[0]: samp_tree.ef
}
xhats, start = sample_tree.walking_tree_xhats(
mname,
local_scenarios,
xhat_one['ROOT'],
branching_factors,
start,
scenario_creator_kwargs,
solvername=solvername,
solver_options=solver_options)
#Compute then the average function value with this policy
scenario_creator_kwargs = samp_tree.ama.kwargs
all_nodenames = sputils.create_nodenames_from_branching_factors(
branching_factors)
else:
#In a 2 stage problem, the only non-leaf is the ROOT node
xhats = xhat_one
all_nodenames = None
xhat_eval_options = {
"iter0_solver_options": None,
"iterk_solver_options": None,
"display_timing": False,
"solvername": solvername,
"verbose": False,
"solver_options": solver_options
}
ev = xhat_eval.Xhat_Eval(xhat_eval_options,
scenario_names,
ama_object.scenario_creator,
scenario_denouement,
scenario_creator_kwargs=scenario_creator_kwargs,
all_nodenames=all_nodenames)
#Evaluating xhat and xstar and getting the value of the objective function
#for every (local) scenario
zhat = ev.evaluate(xhats)
objs_at_xhat = ev.objs_dict
zstar = ev.evaluate(xstars)
objs_at_xstar = ev.objs_dict
eval_scen_at_xhat = []
eval_scen_at_xstar = []
scen_probs = []
for k, s in ev.local_scenarios.items():
eval_scen_at_xhat.append(objs_at_xhat[k])
eval_scen_at_xstar.append(objs_at_xstar[k])
scen_probs.append(s._mpisppy_probability)
scen_gaps = np.array(eval_scen_at_xhat) - np.array(eval_scen_at_xstar)
local_gap = np.dot(scen_gaps, scen_probs)
local_ssq = np.dot(scen_gaps**2, scen_probs)
local_prob_sqnorm = np.linalg.norm(scen_probs)**2
local_obj_at_xhat = np.dot(eval_scen_at_xhat, scen_probs)
local_estim = np.array(
[local_gap, local_ssq, local_prob_sqnorm, local_obj_at_xhat])
global_estim = np.zeros(4)
ev.mpicomm.Allreduce(local_estim, global_estim, op=mpi.SUM)
G, ssq, prob_sqnorm, obj_at_xhat = global_estim
if global_rank == 0 and verbose:
print(f"G = {G}")
sample_var = (ssq - G**2) / (1 - prob_sqnorm) #Unbiased sample variance
s = np.sqrt(sample_var)
use_relative_error = (np.abs(zstar) > 1)
G = correcting_numeric(G,
objfct=obj_at_xhat,
relative_error=use_relative_error)
if objective_gap:
if is_multi:
return {
"G": G,
"s": s,
"zhats": [zhat],
"zstars": [zstar],
"seed": start
}
else:
return {
"G": G,
"s": s,
"zhats": eval_scen_at_xhat,
"zstars": eval_scen_at_xstar,
"seed": start
}
else:
return {"G": G, "s": s, "seed": start}
def walking_tree_xhats(mname,
local_scenarios,
xhat_one,
BFs,
seed,
options,
solvername="gurobi",
solver_options=None):
"""
This methods takes a scenario tree (represented by a scenario list) as an input,
a first stage policy xhat_one and several settings, and computes
a feasible policy for every scenario, i.e. finds nonanticipative xhats
using the SampleSubtree class.
We use a tree traversal approach, so that for every non-leaf node of
the scenario tree we compute an associated sample tree only once.
Parameters
----------
mname : str
name of the module used to sample.
local_scenarios : dict of pyomo.ConcreteModel
Scenarios forming the scenario tree.
xhat_one : list or np.array of float
A feasible and nonanticipative first stage policy.
BFs : list of int
Branching factors for sample trees.
BFs[i] is the branching factor for stage i+2.
seed : int
Starting seed to create scenarios.
options : dict
Arguments passed to the scenario creator.
Returns
-------
xhats : dict
Dict of values for the nonanticipative variable for every node.
keys are node names and values are lists of nonant variables.
NOTE: The local_scenarios do not need to form a regular tree (unbalanced tree are authorized)
"""
if xhat_one is None:
raise RuntimeError("Xhat_one can't be None for now")
xhats = {'ROOT': xhat_one}
#Special case if we only have one scenario
if len(local_scenarios) == 1:
scen = list(local_scenarios.values())[0]
res = feasible_solution(mname,
scen,
xhat_one,
BFs,
seed,
options,
solvername=solvername,
solver_options=solver_options)
return res
for k, s in local_scenarios.items():
scen_xhats = []
ciutils.is_sorted(s._mpisppy_node_list)
for node in s._mpisppy_node_list:
if node.name in xhats:
scen_xhats.append(xhats[node.name])
else:
subtree = SampleSubtree(mname, scen_xhats, s, node.stage, BFs,
seed, options, solvername,
solver_options)
subtree.run()
xhat = subtree.xhat_at_stage
seed += sputils.number_of_nodes(BFs[(node.stage - 1):])
xhats[node.name] = xhat
scen_xhats.append(xhat)
return xhats, seed
ama_options = {
"EF-mstage": True,
"num_scens": num_scens,
"_mpisppy_probability": 1 / num_scens,
"BFs": BFs,
}
#We use from_module to build easily an Amalgomator object
ama = amalgomator.from_module(mname, ama_options, use_command_line=False)
ama.run()
# get the xhat
xhat_one = sputils.nonant_cache_from_ef(ama.ef)['ROOT']
#----------Find a feasible solution for a single scenario-------------
scenario = ama.ef.scen0
seed = sputils.number_of_nodes(BFs)
options = dict() #We take default aircond options
xhats, seed = feasible_solution(mname, scenario, xhat_one, BFs, seed,
options)
print(xhats)
#----------Find feasible solutions for every scenario ------------
#Fetching scenarios from EF
scenarios = dict()
for k in ama.ef._ef_scenario_names:
#print(f"{k =}")
scenarios[k] = getattr(ama.ef, k)
s = scenarios[k]
demands = [s.stage_models[t].Demand for t in s.T]
def run(self, maxit=200):
# Do everything as specified by the options (maxit is provided as a safety net).
refmodel = self.refmodel
mult = self.sample_size_ratio # used to set m_k= mult*n_k
scenario_denouement = refmodel.scenario_denouement if hasattr(
refmodel, "scenario_denouement") else None
#----------------------------Step 0 -------------------------------------#
#Initialization
k = 1
if self.stopping_criterion == "BM":
#Finding a constant used to compute nk
r = 2 #TODO : we could add flexibility here
j = np.arange(1, 1000)
if self.q is None:
s = sum(np.power(j, -self.p * np.log(j)))
else:
if self.q < 1:
raise RuntimeError("Parameter q should be greater than 1.")
s = sum(np.exp(-self.p * np.power(j, 2 * self.q / r)))
self.c = max(
1, 2 * np.log(s / (np.sqrt(2 * np.pi) *
(1 - self.confidence_level))))
lower_bound_k = self.sample_size(k, None, None, None)
#Computing xhat_1.
#We use sample_size_ratio*n_k observations to compute xhat_k
xhat_branching_factors = ciutils.scalable_branching_factors(
mult * lower_bound_k, self.options['branching_factors'])
mk = np.prod(xhat_branching_factors)
self.xhat_gen_options[
'start_seed'] = self.SeedCount #TODO: Maybe find a better way to manage seed
xhat_scenario_names = refmodel.scenario_names_creator(mk)
xgo = self.xhat_gen_options.copy()
xgo.pop("solver_options", None) # it will be given explicitly
xgo.pop("scenario_names", None) # it will be given explicitly
xhat_k = self.xhat_generator(xhat_scenario_names,
solver_options=self.solver_options,
**xgo)
self.SeedCount += sputils.number_of_nodes(xhat_branching_factors)
#----------------------------Step 1 -------------------------------------#
#Computing n_1 and associated scenario names
nk = np.prod(
ciutils.scalable_branching_factors(
lower_bound_k, self.options['branching_factors'])
) #To ensure the same growth that in the one-tree seqsampling
estimator_scenario_names = refmodel.scenario_names_creator(nk)
#Computing G_nk and s_k associated with xhat_1
Gk, sk = self._gap_estimators_with_independent_scenarios(
xhat_k, nk, estimator_scenario_names, scenario_denouement)
#----------------------------Step 2 -------------------------------------#
while (self.stop_criterion(Gk, sk, nk) and k < maxit):
#----------------------------Step 3 -------------------------------------#
k += 1
nk_m1 = nk #n_{k-1}
mk_m1 = mk
lower_bound_k = self.sample_size(k, Gk, sk, nk_m1)
#Computing m_k and associated scenario names
xhat_branching_factors = ciutils.scalable_branching_factors(
mult * lower_bound_k, self.options['branching_factors'])
mk = np.prod(xhat_branching_factors)
self.xhat_gen_options[
'start_seed'] = self.SeedCount #TODO: Maybe find a better way to manage seed
xhat_scenario_names = refmodel.scenario_names_creator(mk)
#Computing xhat_k
xgo = self.xhat_gen_options.copy()
xgo.pop("solver_options", None) # it will be given explicitly
xgo.pop("scenario_names", None) # it will be given explicitly
xhat_k = self.xhat_generator(xhat_scenario_names,
solver_options=self.solver_options,
**xgo)
#Computing n_k and associated scenario names
self.SeedCount += sputils.number_of_nodes(xhat_branching_factors)
nk = np.prod(
ciutils.scalable_branching_factors(
lower_bound_k, self.options['branching_factors'])
) #To ensure the same growth that in the one-tree seqsampling
nk += self.batch_size - nk % self.batch_size
estimator_scenario_names = refmodel.scenario_names_creator(nk)
Gk, sk = self._gap_estimators_with_independent_scenarios(
xhat_k, nk, estimator_scenario_names, scenario_denouement)
if (k % 10 == 0):
print(f"k={k}")
print(f"n_k={nk}")
#----------------------------Step 4 -------------------------------------#
if (k == maxit):
raise RuntimeError(
f"The loop terminated after {maxit} iteration with no acceptable solution"
)
T = k
final_xhat = xhat_k
if self.stopping_criterion == "BM":
upper_bound = self.h * sk + self.eps
elif self.stopping_criterion == "BPL":
upper_bound = self.eps
else:
raise RuntimeError("Only BM and BPL criterion are supported yet.")
CI = [0, upper_bound]
global_toc(f"G={Gk}")
global_toc(f"s={sk}")
global_toc(f"xhat has been computed with {nk*mult} observations.")
return {
"T": T,
"Candidate_solution": final_xhat,
"CI": CI,
}
ama_options = { "EF-mstage": True,
"num_scens": num_scens,
"_mpisppy_probability": 1/num_scens,
"branching_factors":branching_factors,
}
#We use from_module to build easily an Amalgamator object
ama = amalgamator.from_module(mname, ama_options,use_command_line=False)
ama.run()
# get the xhat
xhat_one = sputils.nonant_cache_from_ef(ama.ef)['ROOT']
#----------Find a feasible solution for a single scenario-------------
scenario = ama.ef.scen0
seed = sputils.number_of_nodes(branching_factors)
options = dict() #We take default aircond options
xhats,seed = _feasible_solution(mname, scenario, xhat_one, branching_factors, seed, options)
print(xhats)
#----------Find feasible solutions for every scenario ------------
#Fetching scenarios from EF
scenarios = dict()
for k in ama.ef._ef_scenario_names:
#print(f"{k =}")
scenarios[k] = getattr(ama.ef, k)
s = scenarios[k]
demands = [s.stage_models[t].Demand for t in s.T]
#print(f"{demands =}")
