def test_random_bundles(self):
self.assertTrue("ReferenceModel" not in sys.modules)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(farmer_model_dir,
farmer_scenariotree_dir)
scenario_tree = \
scenario_instance_factory.generate_scenario_tree(random_bundles=2)
self.assertEqual(scenario_tree.contains_bundles(), True)
self.assertEqual(len(scenario_tree._scenario_bundles), 2)
self.assertTrue("ReferenceModel" in sys.modules)
del sys.modules["ReferenceModel"]
def run(args=None):
###################################
print("RUNNING - run args=%s" % str(args))
import pyomo.environ
def LagrangeParametric(args=None):
class Object(object): pass
Result = Object()
Result.status = 'LagrangeParam begins '+ datetime_string() + '...running new ph'
ph = None
blanks = " " # used for formatting print statements
# options used
betaMin = options.beta_min
betaMax = options.beta_max
betaTol = options.beta_tol
gapTol = options.Lagrange_gap
minProb = options.min_prob
maxIntervals = options.max_intervals
maxTime = options.max_time
IndVarName = options.indicator_var_name
multName = options.lambda_parm_name
CCStageNum = options.stage_num
csvPrefix = options.csvPrefix
verbosity = options.verbosity
verbosity = 2 # override for debug (= 3 to get super-debug)
HGdebug = 0 # special debug (not public)
# local...may become option
optTol = gapTol
####################################################################
STARTTIME = time.time()
Result.status = "options set"
if verbosity > 1:
print("From LagrangeParametric, status = %s\tSTARTTIME = %s" \
% (str(getattr(Result,'status')), str(STARTTIME)))
ph = PHFromScratch(options)
Result.ph = ph
rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios
ReferenceInstance = ph._instances[rootnode._scenarios[0]._name] # arbitrary scenario
if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
sense = 'min'
else:
sense = 'max'
scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)
if options.verbosity > 0: print("%s %s scenarios" % (str(sense),str(scenario_count)))
# initialize
Result.status = 'starting at '+datetime_string()
if verbosity > 0:
print(Result.status)
ScenarioList = []
lambdaval = 0.
lagrUtil.Set_ParmValue(ph, multName,lambdaval)
# IMPORTANT: Preprocess the scenario instances
# before fixing variables, otherwise they
# will be preprocessed out of the expressions
# and the output_fixed_variable_bounds option
# will have no effect when we update the
# fixed variable values (and then assume we
# do not need to preprocess again because
# of this option).
ph._preprocess_scenario_instances()
sumprob = 0.
minprob = 1.
maxprob = 0.
# fixed = 0 to get PR point at b=0
lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0)
for scenario in rootnode._scenarios:
instance = ph._instances[scenario._name]
sname = scenario._name
sprob = scenario._probability
sumprob = sumprob + sprob
minprob = min(minprob,sprob)
maxprob = max(maxprob,sprob)
ScenarioList.append([sname,sprob])
ScenarioList.sort(key=operator.itemgetter(1)) # sorts from min to max probability
if verbosity > 0:
print("probabilities sum to %f range: %f to %f" % (sumprob,minprob,maxprob))
Result.ScenarioList = ScenarioList
# Write ScenarioList = name, probability in csv file sorted by probability
outName = csvPrefix + 'ScenarioList.csv'
print("writing to %s" % outName)
with open(outName,'w') as outFile:
for scenario in ScenarioList:
outFile.write(scenario[0]+", "+str(scenario[1])+'\n')
Result.ScenarioList = ScenarioList
addstatus = 'Scenario List written to ' + csvPrefix+'ScenarioList.csv'
Result.status = Result.status + '\n' + addstatus
if verbosity > 0:
print(addstatus)
if verbosity > 0:
print("solve begins %s" % datetime_string())
print("\t- lambda = %f" % lambdaval)
SolStat, zL = lagrUtil.solve_ph_code(ph, options)
if verbosity > 0:
print("solve ends %s" % datetime_string())
print("\t- status = %s" % str(SolStat))
print("\t- zL = %s" % str(zL))
bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
if bL > 0:
print("** bL = %s > 0 (all %s = 0)" % (str(bL), str(IndVarName)))
return Result
if verbosity > 0: print("Initial optimal obj = %s for bL = %s" % (str(zL), str(bL)))
# fixed = 1 to get PR point at b=1
lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1)
if verbosity > 0:
print("solve begins %s" % datetime_string())
print("\t- lambda = %s" % str(lambdaval))
SolStat, zU = lagrUtil.solve_ph_code(ph, options)
if verbosity > 0:
print("solve ends %s" % datetime_string())
print("\t- status = %s" % str(SolStat))
print("\t- zU = %s" % str(zU))
if not SolStat[0:2] == 'ok':
print(str(SolStat[0:3])+" is not 'ok'")
addstatus = "** Solution is non-optimal...aborting"
print(addstatus)
Result.status = Result.status + "\n" + addstatus
return Result
bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
if bU < 1.- betaTol and verbosity > 0:
print("** Warning: bU = %s < 1" % str(bU))
### enumerate points in PR space (all but one scenario)
# Result.lbz = [ [0,bL,zL], [None,bU,zU] ]
# for scenario in rootnode._scenarios:
# sname = scenario._name
# instance = ph._instances[sname]
# print "excluding scenario",sname
# getattr(instance,IndVarName).value = 0
# print sname,"value =",getattr(instance,IndVarName).value,getattr(instance,IndVarName).fixed
# SolStat, z = lagrUtil.solve_ph_code(ph, options)
# b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
# print "solve ends with status =",SolStat,"(b, z) =",b,z
# getattr(instance,IndVarName).value = 1
# Result.lbz.append([None,b,z])
# for t in instance.TimePeriods:
# print "Global at",t,"=",instance.posGlobalLoadGenerateMismatch[t].value, \
# '-',instance.negGlobalLoadGenerateMismatch[t].value,"=",\
# instance.GlobalLoadGenerateMismatch[t].value,\
# "\tDemand =",instance.TotalDemand[t].value, ",",\
# "Reserve =",instance.ReserveRequirement[t].value
#
# PrintPRpoints(Result.lbz)
# return Result
#### end enumeration
########################################################################
if verbosity > 1:
print("We have bU = %s ...about to free all %s for %d scenarios" % \
(str(bU), str(IndVarName), len(ScenarioList)))
# free scenario selection variable
lagrUtil.FreeAllIndicatorVariables(ph, IndVarName)
if verbosity > 1:
print("\tall %s freed; elapsed time = %f" % (str(IndVarName), time.time() - STARTTIME))
# initialize with the two endpoints
Result.lbz = [ [0.,bL,zL], [None,bU,zU] ]
Result.selections = [[], ScenarioList]
NumIntervals = 1
if verbosity > 0:
print("Initial relative Lagrangian gap = %f maxIntervals = %d" % (1-zL/zU, maxIntervals))
if verbosity > 1:
print("entering while loop %s" % datetime_string())
print("\n")
############ main loop to search intervals #############
########################################################
while NumIntervals < maxIntervals:
lapsedTime = time.time() - STARTTIME
if lapsedTime > maxTime:
addstatus = '** max time reached ' + str(lapsedTime)
print(addstatus)
Result.status = Result.status + '\n' + addstatus
break
if verbosity > 1:
print("Top of while with %d intervals elapsed time = %f" % (NumIntervals, lapsedTime))
PrintPRpoints(Result.lbz)
lambdaval = None
### loop over PR points to find first unfathomed interval to search ###
for PRpoint in range(1,len(Result.lbz)):
if Result.lbz[PRpoint][0] == None:
# multiplier = None means interval with upper endpoint at PRpoint not fathomed
bL = Result.lbz[PRpoint-1][1]
zL = Result.lbz[PRpoint-1][2]
bU = Result.lbz[PRpoint][1]
zU = Result.lbz[PRpoint][2]
lambdaval = (zU - zL) / (bU - bL)
break
#############################
# Exited from the for loop
if verbosity > 1:
print("exited for loop with PRpoint = %s ...lambdaval = %s" % (PRpoint, lambdaval))
if lambdaval == None: break # all intervals are fathomed
if verbosity > 1: PrintPRpoints(Result.lbz)
if verbosity > 0:
print("Searching for b in [%s, %s] with %s = %f" % (str(round(bL,4)), str(round(bU,4)), multName, lambdaval))
# search interval (bL,bU)
lagrUtil.Set_ParmValue(ph, multName,lambdaval)
if verbosity > 0:
print("solve begins %s" % datetime_string())
print("\t- %s = %f" % (multName, lambdaval))
#########################################################
SolStat, Lagrangian = lagrUtil.solve_ph_code(ph, options)
#########################################################
if not SolStat[0:2] == 'ok':
addstatus = "** Solution status " + SolStat + " is not optimal"
print(addstatus)
Result.status = Result.status + "\n" + addstatus
return Result
b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
z = Lagrangian + lambdaval*b
if verbosity > 0:
print("solve ends %s" % datetime_string())
print("\t- Lagrangian = %f" % Lagrangian)
print("\t- b = %s" % str(b))
print("\t- z = %s" % str(z))
print("\n")
# We have PR point (b,z), which may be new or one of the endpoints
##################################################################
######### Begin tolerance tests ##########
# Test that b is in [bL,bU]
if verbosity > 1: print("\ttesting b")
if b < bL - betaTol or b > bU + betaTol:
addstatus = "** fatal error: probability (= " + str(b) + \
") is outside interval, (" + str(bL) + ", " + str(bU) + ")"
addstatus = addstatus + "\n\t(tolerance = " + str(betaTol) + ")"
print(addstatus+'\n')
Result.status = Result.status + addstatus
return Result
# Test that z is in [zL,zU]
if verbosity > 1: print("\ttesting z")
# using optTol as absolute tolerance (not relative)
# ...if we reconsider, need to allow negative z-values
if z < zL - optTol or z > zU + optTol:
addstatus = "** fatal error: obj (= " + str(z) + \
") is outside interval, (" + str(zL) + ", " + str(zU) + ")"
print(addstatus+'\n')
Result.status = Result.status + addstatus
return Result
# Ok, we have (b,z) in [(bL,zL), (bU,zU)], at least within tolerances
oldLagrangian = zL - lambdaval*bL
# ensure lambdaval set such that endpoints have same Lagrangian value
# (this is probably unnecessary, but check anyway)
if abs(oldLagrangian - (zU - lambdaval*bU)) > optTol*abs(oldLagrangian):
addstatus = "** fatal error: Lagrangian at (bL,zL) = " + \
str(oldLagrangian) + " not= " + str(zU-lambdaval*bU) + \
"\n\t(optTol = " + str(optTol) + ")"
Result.status = Result.status + addstatus
return Result
# no more fatal error tests...need to know if (b,z) is endpoint or new
if verbosity > 1: print("No anomalies...testing if b = bL or bU")
# Test if endpoint is an alternative optimum of Lagrangian
# ...using optTol as *relative* tolerance
# (could use other reference values -- eg, avg or max of old and new Lagrangian values)
refValue = max( min( abs(oldLagrangian), abs(Lagrangian) ), 1.)
alternativeOpt = abs( oldLagrangian - Lagrangian ) <= optTol*refValue
# alternativeOpt = True means we computed point (b,z) is alternative optimum such that:
# case 1: (b,z) = endpoint, in which case we simply fathom [bL,bU] by setting PRpoint
# to [lambdaval,bU,zU] (the numeric value of multiplier means fathomed)
# case 2: (b,z) is new PR point on line segment, in which case we split into
# [bL,b] and [b,bU], with both fathomed
if verbosity > 1:
print("oldLagrangian = %s" % str(oldLagrangian))
if alternativeOpt: print(":= Lagrangian = %s" % str(Lagrangian ))
else: print("> Lagrangian = %s" % str(Lagrangian))
if alternativeOpt:
# setting multiplier of (bU,zU) to a numeric fathoms the interval [bL,bU]
Result.lbz[PRpoint][0] = lambdaval
# test if (b,z) is an endpoint
newPRpoint = abs(b-bL) > betaTol and abs(b-bU) > betaTol
if not newPRpoint:
# ...(b,z) is NOT an endpoint (or sufficiently close), so split and fathom
if verbosity > 1:
print("\tnot an endpoint\tlbz = %s" % str(Result.lbz[PRpoint]))
if verbosity > 0:
print("Lagangian solution is new PR point on line segment of (" \
+ str(bL) + ", " + str(bU) +")")
print("\tsplitting (bL,bU) into (bL,b) and (b,bU), both fathomed")
# note: else ==> b = bL or bU, so we do nothing, having already fathomed [bL,bU]
# (b,z) is new PR point, so split interval (still in while loop)
##########################################
# alternative optimum ==> split & fathom: (bL,b), (b,bU)
if verbosity > 1:
print("\talternativeOpt %s newPRpoint = %s" % (alternativeOpt, newPRpoint))
if newPRpoint:
NumIntervals += 1
if alternativeOpt:
if verbosity > 1: print("\tInsert [lambdaval,b,z] at %f" % PRpoint)
Result.lbz = Insert([lambdaval,b,z],PRpoint,Result.lbz)
addstatus = "Added PR point on line segment of envelope"
if verbosity > 0: print(addstatus+'\n')
else:
if verbosity > 1: print("\tInsert [None,b,z] at %f" % PRpoint)
Result.lbz = Insert([None,b,z],PRpoint,Result.lbz)
addstatus = "new envelope extreme point added (interval split, not fathomed)"
Result.status = Result.status + "\n" + addstatus
if verbosity > 1:
print("...after insertion:")
PrintPRpoints(Result.lbz)
# get the selections of new point (ie, scenarios for which delta=1)
Selections = []
for scenario in ScenarioList:
instance = ph._instances[scenario[0]]
if getattr(instance,IndVarName).value == 1:
Selections.append(scenario)
Result.selections = Insert(Selections,PRpoint,Result.selections)
if verbosity > 0:
print("Interval "+str(PRpoint)+", ["+str(bL)+", "+str(bU)+ \
"] split at ("+str(b)+", "+str(z)+")")
print("\tnew PR point has "+str(len(Selections))+" selections")
if verbosity > 1: print("test that selections list aligned with lbz")
if not len(Result.lbz) == len(Result.selections):
print("** fatal error: lbz not= selections")
PrintPRpoints(Result.lbz)
print("Result.selections:")
for i in range(Result.selections): print("%d %f" % (i,Result.selections[i]))
return Result
# ok, we have split and/or fathomed interval
if NumIntervals >= maxIntervals:
# we are about to leave while loop due to...
addstatus = "** terminating because number of intervals = " + \
str(NumIntervals) + " >= max = " + str(maxIntervals)
if verbosity > 0: print(addstatus+'\n')
Result.status = Result.status + "\n" + addstatus
# while loop continues
if verbosity > 1:
print("bottom of while loop")
PrintPRpoints(Result.lbz)
###################################################
# end while NumIntervals < maxIntervals:
# ^ this is indentation of while loop
################ end while loop ###################
if verbosity > 1: print("\nend while loop...setting multipliers")
for i in range(1,len(Result.lbz)):
db = Result.lbz[i][1] - Result.lbz[i-1][1]
dz = Result.lbz[i][2] - Result.lbz[i-1][2]
if dz > 0:
Result.lbz[i][0] = dz/db
else:
#print "dz =",dz," at ",i,": ",Result.lbz[i]," -",Result.lbz[i-1]
Result.lbz[i][0] = 0
if verbosity > 0: PrintPRpoints(Result.lbz)
addstatus = '\nLagrange multiplier search ends'+datetime_string()
if verbosity > 0:
print(addstatus+'\n')
Result.status = Result.status + addstatus
outName = csvPrefix + "PRoptimal.csv"
with open(outName,'w') as outFile:
if verbosity > 0:
print("writing PR points to "+outName+'\n')
for lbz in Result.lbz:
outFile.write(str(lbz[1])+ ", " +str(lbz[2])+'\n')
outName = csvPrefix + "OptimalSelections.csv"
with open(outName,'w') as outFile:
if verbosity > 0:
print("writing optimal selections for each PR point to "+csvPrefix+'PRoptimal.csv\n')
for selections in Result.selections:
char = ""
thisSelection = ""
for slist in selections:
if slist:
thisSelection = thisSelection + char + slist[0]
char = ","
outFile.write(thisSelection+'\n')
if verbosity > 0:
print("\nReturning status:\n %s \n=======================" % Result.status)
################################
if verbosity > 2:
print("\nAbout to return...Result attributes: %d" % len(inspect.getmembers(Result)))
for attr in inspect.getmembers(Result): print(attr[0])
print("\n===========================================")
# LagrangeParametric ends here
return Result
################################
####################################### start run ####################################
AllInOne = False
########################
# options defined here
########################
try:
conf_options_parser = construct_ph_options_parser("lagrange [options]")
conf_options_parser.add_argument("--beta-min",
help="The min beta level for the chance constraint. Default is 0",
action="store",
dest="beta_min",
type=float,
default=0.)
conf_options_parser.add_argument("--beta-max",
help="The beta level for the chance constraint. Default is 1.",
action="store",
dest="beta_max",
type=float,
default=1.)
conf_options_parser.add_argument("--beta-tol",
help="Tolerance for testing equality to beta. Default is 1e-5",
action="store",
dest="beta_tol",
type=float,
default=1e-5)
conf_options_parser.add_argument("--Lagrange-gap",
help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4",
action="store",
type=float,
dest="Lagrange_gap",
default=0.0001)
conf_options_parser.add_argument("--min-prob",
help="Tolerance for testing probability > 0. Default is 1e-9",
action="store",
dest="min_prob",
type=float,
default=1e-5)
conf_options_parser.add_argument("--max-intervals",
help="The max number of intervals generated; if causes termination, non-fathomed intervals have multiplier=None. Default = 100.",
action="store",
dest="max_intervals",
type=int,
default=100)
conf_options_parser.add_argument("--max-time",
help="Maximum time (seconds). Default is 3600.",
action="store",
dest="max_time",
type=float,
default=3600)
conf_options_parser.add_argument("--lambda-parm-name",
help="The name of the lambda parameter in the model. Default is lambdaMult",
action="store",
dest="lambda_parm_name",
type=str,
default="lambdaMult")
conf_options_parser.add_argument("--indicator-var-name",
help="The name of the indicator variable for the chance constraint. The default is delta",
action="store",
dest="indicator_var_name",
type=str,
default="delta")
conf_options_parser.add_argument("--stage-num",
help="The stage number of the CC indicator variable (number, not name). Default is 2",
action="store",
dest="stage_num",
type=int,
default=2)
conf_options_parser.add_argument("--csvPrefix",
help="Output file name. Default is ''",
action="store",
dest="csvPrefix",
type=str,
default='')
conf_options_parser.add_argument("--verbosity",
help="verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.",
action="store",
dest="verbosity",
type=int,
default=1)
# The following needed for solve_ph_code in lagrangeutils
conf_options_parser.add_argument("--solve-with-ph",
help="Perform solves via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_with_ph",
default=False)
##HG: deleted params filed as deletedParam.py
#######################################################################################################
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
# create the reference instances and the scenario tree - no
# scenario instances yet.
if options.verbosity > 0:
print("Loading reference model and scenario tree")
# Dec 18
# scenario_instance_factory, full_scenario_tree = load_models(options)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(options.model_directory,
options.instance_directory)
full_scenario_tree = \
GenerateScenarioTreeForPH(options,
scenario_instance_factory)
####
try:
if (scenario_instance_factory is None) or (full_scenario_tree is None):
raise RuntimeError("***ERROR: Failed to initialize the model and/or scenario tree data.")
# load_model gets called again, so lets make sure unarchived directories are used
options.model_directory = scenario_instance_factory._model_filename
options.instance_directory = scenario_instance_factory._scenario_tree_filename
########## Here is where multiplier search is called from run() ############
Result = LagrangeParametric()
#####################################################################################
finally:
# delete temporary unarchived directories
scenario_instance_factory.close()
if options.verbosity > 0:
print("\n===========================================")
print("\nreturned from LagrangeParametric")
if options.verbosity > 2:
print("\nFrom run, Result should have status and ph objects...")
for attr in inspect.getmembers(Result): print(attr)
print("\n===========================================")
try:
status = Result.status
print("status = "+str(Result.status))
except:
print("status not defined")
sys.exit()
try:
lbz = Result.lbz
PrintPRpoints(lbz)
with open(options.csvPrefix+"PRoptimal.csv",'w') as outFile:
for lbz in Result.lbz:
outFile.write(str(lbz[1])+ ", " +str(lbz[2])+'\n')
except:
print("Result.lbz not defined")
sys.exit()
try:
ScenarioList = Result.ScenarioList
ScenarioList.sort(key=operator.itemgetter(1))
with open(options.csvPrefix+"ScenarioList.csv",'w') as outFile:
for scenario in ScenarioList:
outFile.write(scenario[0]+", "+str(scenario[1])+'\n')
except:
print("Result.ScenarioList not defined")
sys.exit()
def run(args=None):
AllInOne = False
# The value of AllInOne will be set to True for the "old"
# computeconf (with fraction_for_solve) and will stay False for
# the "new" computeconf (with MRP_directory_basename)
try:
conf_options_parser = construct_ph_options_parser("computeconf [options]")
conf_options_parser.add_argument(
"--fraction-scenarios-for-solve",
help="The fraction of scenarios that are allocated to finding a solution. Default is None.",
action="store",
dest="fraction_for_solve",
type=float,
default=None,
)
conf_options_parser.add_argument(
"--number-samples-for-confidence-interval",
help="The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.",
action="store",
dest="n_g",
type=int,
default=None,
)
conf_options_parser.add_argument(
"--confidence-interval-alpha",
help="The alpha level for the confidence interval. Default is 0.05",
action="store",
dest="confidence_interval_alpha",
type=float,
default=0.05,
)
conf_options_parser.add_argument(
"--solve-xhat-with-ph",
help="Perform xhat solve via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_xhat_with_ph",
default=False,
)
conf_options_parser.add_argument(
"--random-seed",
help="Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.",
action="store",
dest="random_seed",
type=int,
default=0,
)
conf_options_parser.add_argument(
"--append-file",
help="File to which summary run information is appended, for output tracking purposes.",
action="store",
dest="append_file",
type=str,
default=None,
)
conf_options_parser.add_argument(
"--write-xhat-solution",
help="Write xhat solutions (first stage variables only) to the append file? Defaults to False.",
action="store_true",
dest="write_xhat_solution",
default=False,
)
conf_options_parser.add_argument(
"--generate-weighted-cvar",
help="Add a weighted CVaR term to the primary objective",
action="store_true",
dest="generate_weighted_cvar",
default=False,
)
conf_options_parser.add_argument(
"--cvar-weight",
help="The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.",
action="store",
dest="cvar_weight",
type=float,
default=1.0,
)
conf_options_parser.add_argument(
"--risk-alpha",
help="The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.",
action="store",
dest="risk_alpha",
type=float,
default=0.95,
)
conf_options_parser.add_argument(
"--MRP-directory-basename",
help="The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None",
action="store",
dest="MRP_directory_basename",
type=str,
default=None,
)
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
# seed the generator if a user-supplied seed is
# provided. otherwise, python will seed from the current system
# time.
if options.random_seed > 0:
random.seed(options.random_seed)
start_time = time.time()
if options.verbose:
print("Importing model and scenario tree files")
scenario_instance_factory = ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory)
if _OLD_OUTPUT:
print("Loading reference model and scenario tree")
if options.verbose or options.output_times:
print("Time to import model and scenario " "tree structure files=%.2f seconds" % (time.time() - start_time))
try:
scenario_tree = scenario_instance_factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed,
verbose=options.verbose,
)
#
# print the input tree for validation/information purposes.
#
if options.verbose:
scenario_tree.pprint()
#
# validate the tree prior to doing anything serious
#
if not scenario_tree.validate():
raise RuntimeError("Scenario tree is invalid")
else:
if options.verbose:
print("Scenario tree is valid!")
index_list, num_scenarios_for_solution, num_scenarios_per_sample = partition_scenario_space(
scenario_tree, options
)
# index_list = [0,3,5,7,1,4,6,8,2,9]
# for ndx in index_list:
# print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name))
xhat_ph = find_candidate(
scenario_instance_factory, index_list, num_scenarios_for_solution, scenario_tree, options
)
run_conf(
scenario_instance_factory,
index_list,
num_scenarios_for_solution,
num_scenarios_per_sample,
scenario_tree,
xhat_ph,
options,
)
finally:
# delete temporary unarchived directories
if scenario_instance_factory is not None:
scenario_instance_factory.close()
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
_declared_options = \
PySPConfigBlock("Options declared for the "
"ScenarioTreeServerPyro class")
#
# scenario instance construction
#
safe_declare_common_option(_declared_options,
"model_location")
safe_declare_common_option(_declared_options,
"scenario_tree_location")
#
# scenario tree generation
#
safe_declare_common_option(_declared_options,
"scenario_tree_random_seed")
safe_declare_common_option(_declared_options,
"scenario_tree_downsample_fraction")
#
# various
#
safe_declare_common_option(_declared_options,
"verbose")
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
% (os.getpid(), socket.gethostname()))
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
TaskWorker.__init__(self, **kwds)
# This classes options get updated during the "setup" phase
options = self.register_options()
PySPConfiguredObject.__init__(self, options)
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
return self._process(data)
except:
logger.error(
"Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle."
% (self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return TaskProcessingError(traceback.format_exc())
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
options = self.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
self.set_options(options)
self._options.verbose = self._options.verbose | self._verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._options.verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
print("Options:")
self.display_options()
# Make sure these are not archives
assert os.path.exists(self._options.model_location)
assert (self._options.scenario_tree_location is None) or \
os.path.exists(self._options.scenario_tree_location)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
self._options.model_location,
scenario_tree_location=self._options.scenario_tree_location,
verbose=self._options.verbose)
#
# Try prevent unnecessarily re-imported the model module
# if other callbacks are in the same location
#
self._modules_imported[
self._scenario_instance_factory._model_location] = \
self._scenario_instance_factory._model_module
self._modules_imported[
self._scenario_instance_factory._model_filename] = \
self._scenario_instance_factory._model_module
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree(
downsample_fraction=self._options.scenario_tree_downsample_fraction,
random_seed=self._options.scenario_tree_random_seed)
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._options.verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
options = worker_type.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
#
# Depending on the Pyro serializer, the namedtuple
# may be been converted to a tuple
#
if not isinstance(data.worker_init, WorkerInit):
assert type(data.worker_init) is tuple
data.worker_init = WorkerInit(type_=data.worker_init[0],
names=data.worker_init[1],
data=data.worker_init[2])
# replace enum string representation with the actual enum
# object now that we've unserialized the Pyro data
worker_init = WorkerInit(type_=getattr(WorkerInitType,
data.worker_init.type_),
names=data.worker_init.names,
data=data.worker_init.data)
self._worker_map[worker_name] = worker_type(options)
self._worker_map[worker_name].initialize(
self.WORKERNAME,
self._full_scenario_tree,
worker_name,
worker_init,
self._modules_imported)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._options.verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._options.verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._options.verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
options = self.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
self.set_options(options)
self._options.verbose = self._options.verbose | self._verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._options.verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
print("Options:")
self.display_options()
# Make sure these are not archives
assert os.path.exists(self._options.model_location)
assert (self._options.scenario_tree_location is None) or \
os.path.exists(self._options.scenario_tree_location)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
self._options.model_location,
scenario_tree_location=self._options.scenario_tree_location,
verbose=self._options.verbose)
#
# Try prevent unnecessarily re-imported the model module
# if other callbacks are in the same location
#
self._modules_imported[
self._scenario_instance_factory._model_location] = \
self._scenario_instance_factory._model_module
self._modules_imported[
self._scenario_instance_factory._model_filename] = \
self._scenario_instance_factory._model_module
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree(
downsample_fraction=self._options.scenario_tree_downsample_fraction,
random_seed=self._options.scenario_tree_random_seed)
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._options.verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
options = worker_type.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
#
# Depending on the Pyro serializer, the namedtuple
# may be been converted to a tuple
#
if not isinstance(data.worker_init, WorkerInit):
assert type(data.worker_init) is tuple
data.worker_init = WorkerInit(type_=data.worker_init[0],
names=data.worker_init[1],
data=data.worker_init[2])
# replace enum string representation with the actual enum
# object now that we've unserialized the Pyro data
worker_init = WorkerInit(type_=getattr(WorkerInitType,
data.worker_init.type_),
names=data.worker_init.names,
data=data.worker_init.data)
self._worker_map[worker_name] = worker_type(options)
self._worker_map[worker_name].initialize(
self.WORKERNAME,
self._full_scenario_tree,
worker_name,
worker_init,
self._modules_imported)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._options.verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._options.verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._options.verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
def run(args=None):
###################################
# to import plugins
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
def LagrangeMorePR(args=None):
print("lagrangeMorePR begins %s" % datetime_string())
blanks = " " # used for formatting print statements
class Object(object): pass
Result = Object()
# options used
betaTol = options.beta_tol # tolerance used to separate b-values
IndVarName = options.indicator_var_name
multName = options.lambda_parm_name
CCStageNum = options.stage_num
MaxMorePR = options.max_number # max PR points to be generated (above F^* with all delta fixed)
MaxTime = options.max_time # max time before terminate
csvPrefix = options.csvPrefix # input filename prefix (eg, case name)
probFileName = options.probFileName # name of file containing probabilities
##HG override
# options.verbosity = 2
verbosity = options.verbosity
Result.status = 'starting '+datetime_string()
STARTTIME = time.time()
ph = PHFromScratch(options)
rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios
if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
print("We are solving a MINIMIZATION problem.")
else:
print("We are solving a MAXIMIZATION problem.")
# initialize
ScenarioList = []
with open(csvPrefix+"ScenarioList.csv",'r') as inputFile:
for line in inputFile.readlines():
L = line.split(',')
ScenarioList.append([L[0],float(L[1])])
addstatus = str(len(ScenarioList))+' scenarios read from file: ' + csvPrefix+'ScenarioList.csv'
if verbosity > 0: print(addstatus)
Result.status = Result.status + '\n' + addstatus
PRoptimal = []
with open(csvPrefix+"PRoptimal.csv",'r') as inputFile:
for line in inputFile.readlines():
bzS = line.split(',')
PRoptimal.append( [None, float(bzS[0]), float(bzS[1])] )
addstatus = str(len(PRoptimal))+' PR points read from file: '+ csvPrefix+'PRoptimal.csv (envelope function)'
if verbosity > 0:
print(addstatus)
Result.status = Result.status + '\n' + addstatus
# ensure PR points on envelope function are sorted by probability
PRoptimal.sort(key=operator.itemgetter(1))
PRoptimal[0][0] = 0 # initial lambda (for b=0)
for p in range(1,len(PRoptimal)):
dz = PRoptimal[p][2] - PRoptimal[p-1][2]
db = PRoptimal[p][1] - PRoptimal[p-1][1]
PRoptimal[p][0] = dz/db
if verbosity > 0:
PrintPRpoints(PRoptimal)
Result.PRoptimal = PRoptimal
lambdaval = 0.
lagrUtil.Set_ParmValue(ph, options.lambda_parm_name,lambdaval)
# IMPORTANT: Preprocess the scenario instances
# before fixing variables, otherwise they
# will be preprocessed out of the expressions
# and the output_fixed_variable_bounds option
# will have no effect when we update the
# fixed variable values (and then assume we
# do not need to preprocess again because
# of this option).
ph._preprocess_scenario_instances()
## read scenarios to select for each PR point on envelope function
with open(csvPrefix+"OptimalSelections.csv",'r') as inputFile:
OptimalSelections = []
for line in inputFile.readlines():
if len(line) == 0: break # eof
selections = line.split(',')
L = len(selections)
Ls = len(selections[L-1])
selections[L-1] = selections[L-1][0:Ls-1]
if verbosity > 1:
print(str(selections))
OptimalSelections.append(selections)
Result.OptimalSelections = OptimalSelections
addstatus = str(len(OptimalSelections)) + ' Optimal selections read from file: ' \
+ csvPrefix + 'OptimalSelections.csv'
Result.status = Result.status + '\n' + addstatus
if len(OptimalSelections) == len(PRoptimal):
if verbosity > 0:
print(addstatus)
else:
addstatus = addstatus + '\n** Number of selections not equal to number of PR points'
print(addstatus)
Result.status = Result.status + '\n' + addstatus
print(str(OptimalSelections))
print((PRoptimal))
return Result
#####################################################################################
# get probabilities
if probFileName is None:
# ...generate from widest gap regions
PRlist = FindPRpoints(options, PRoptimal)
else:
# ...read probabilities
probList = []
with open(probFileName,'r') as inputFile:
if verbosity > 0:
print("reading from probList = "+probFileName)
for line in inputFile.readlines(): # 1 probability per line
if len(line) == 0:
break
prob = float(line)
probList.append(prob)
if verbosity > 0:
print("\t "+str(len(probList))+" probabilities")
if verbosity > 1:
print(str(probList))
PRlist = GetPoints(options, PRoptimal, probList)
if verbosity > 1:
print("PRlist:")
for interval in PRlist:
print(str(interval))
# We now have PRlist = [[i, b], ...], where b is in PRoptimal interval (i-1,i)
addstatus = str(len(PRlist)) + ' probabilities'
if verbosity > 0:
print(addstatus)
Result.status = Result.status + '\n' + addstatus
#####################################################################################
lapsedTime = time.time() - STARTTIME
addstatus = 'Initialize complete...lapsed time = ' + str(lapsedTime)
if verbosity > 1:
print(addstatus)
Result.status = Result.status + '\n' + addstatus
#####################################################################################
if verbosity > 1:
print("\nlooping over Intervals to generate PR points by flipping heuristic")
Result.morePR = []
for interval in PRlist:
lapsedTime = time.time() - STARTTIME
if lapsedTime > MaxTime:
addstatus = '** lapsed time = ' + str(lapsedTime) + ' > max time = ' + str(MaxTime)
if verbosity > 0: print(addstatus)
Result.status = Result.status + '\n' + addstatus
break
i = interval[0] # = PR point index
b = interval[1] # = target probability to reach by flipping from upper endpoint
bU = PRoptimal[i][1] # = upper endpoint
bL = PRoptimal[i-1][1] # = lower endpoint
if verbosity > 1:
print( "target probability = "+str(b)+" < bU = PRoptimal[" + str(i) + "][1]" \
" and > bL = PRoptimal["+str(i-1)+"][1]")
if b < bL or b > bU:
addstatus = '** probability = '+str(b) + ', not in gap interval: (' \
+ str(bL) + ', ' + str(bU) + ')'
print(addstatus)
print(str(PRoptimal))
print(str(PRlist))
Result.status = Result.status + '\n' + addstatus
return Result
if verbosity > 1:
print( "i = "+str(i)+" : Starting with bU = "+str(bU)+" having "+ \
str(len(OptimalSelections[i]))+ " selections:")
print(str(OptimalSelections[i]))
# first fix all scenarios = 0
for sname, sprob in ScenarioList:
scenario = ph._scenario_tree.get_scenario(sname)
lagrUtil.FixIndicatorVariableOneScenario(ph,
scenario,
IndVarName,
0)
# now fix optimal selections = 1
for sname in OptimalSelections[i]:
scenario = ph._scenario_tree.get_scenario(sname)
lagrUtil.FixIndicatorVariableOneScenario(ph,
scenario,
IndVarName,
1)
# flip scenario selections from bU until we reach b (target probability)
bNew = bU
for sname, sprob in ScenarioList:
scenario = ph._scenario_tree.get_scenario(sname)
if bNew - sprob < b:
continue
instance = ph._instances[sname]
if getattr(instance, IndVarName).value == 0:
continue
bNew = bNew - sprob
# flipped scenario selection
lagrUtil.FixIndicatorVariableOneScenario(ph,
scenario,
IndVarName,
0)
if verbosity > 1:
print("\tflipped "+sname+" with prob = "+str(sprob)+" ...bNew = "+str(bNew))
if verbosity > 1:
print("\tflipped selections reach "+str(bNew)+" >= target = "+str(b)+" (bL = "+str(bL)+")")
if bNew <= bL + betaTol or bNew >= bU - betaTol:
if verbosity > 0:
print("\tNot generating PR point...flipping from bU failed")
continue # to next interval in list
# ready to solve to get cost for fixed scenario selections associated with probability = bNew
if verbosity > 1:
# check that scenarios are fixed as they should be
totalprob = 0.
for scenario in ScenarioList:
sname = scenario[0]
sprob = scenario[1]
instance = ph._instances[sname]
print("fix "+sname+" = "+str(getattr(instance,IndVarName).value)+\
" is "+str(getattr(instance,IndVarName).fixed)+" probability = "+str(sprob))
if getattr(instance,IndVarName).value == 1:
totalprob = totalprob + sprob
lambdaval = getattr(instance, multName).value
print("\ttotal probability = %f" % totalprob)
# solve (all delta fixed); lambda=0, so z = Lagrangian
if verbosity > 0:
print("solve begins %s" % datetime_string())
print("\t- lambda = %f" % lambdaval)
SolStat, z = lagrUtil.solve_ph_code(ph, options)
b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
if verbosity > 0:
print("solve ends %s" % datetime_string())
print("\t- SolStat = %s" % str(SolStat))
print("\t- b = %s" % str(b))
print("\t- z = %s" % str(z))
print("(adding to more PR points)")
Result.morePR.append([None,b,z])
if verbosity > 1:
PrintPRpoints(Result.morePR)
######################################################
# end loop over target probabilities
with open(csvPrefix+"PRmore.csv",'w') as outFile:
for point in Result.morePR:
outFile.write(str(point[1])+','+str(point[2]))
addstatus = str(len(Result.morePR)) + ' PR points written to file: '+ csvPrefix + 'PRmore.csv'
if verbosity > 0: print(addstatus)
Result.status = Result.status + '\n' + addstatus
addstatus = 'lapsed time = ' + putcommas(time.time() - STARTTIME)
if verbosity > 0: print(addstatus)
Result.status = Result.status + '\n' + addstatus
return Result
################################
# LagrangeMorePR ends here
################################
#### start run ####
AllInOne = False
# VERYSTARTTIME=time.time()
# print "##############VERYSTARTTIME:",str(VERYSTARTTIME-VERYSTARTTIME)
##########################
# options defined here
##########################
try:
conf_options_parser = construct_ph_options_parser("lagrange [options]")
conf_options_parser.add_argument("--beta-min",
help="The min beta level for the chance constraint. Default is None",
action="store",
dest="beta_min",
type=float,
default=None)
conf_options_parser.add_argument("--beta-max",
help="The beta level for the chance constraint. Default is None",
action="store",
dest="beta_max",
type=float,
default=None)
conf_options_parser.add_argument("--min-prob",
help="Tolerance for testing probability > 0. Default is 1e-5",
action="store",
dest="min_prob",
type=float,
default=1e-5)
conf_options_parser.add_argument("--beta-tol",
help="Tolerance for testing equality to beta. Default is 10^-2",
action="store",
dest="beta_tol",
type=float,
default=1e-2)
conf_options_parser.add_argument("--Lagrange-gap",
help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4.",
action="store",
type=float,
dest="Lagrange_gap",
default=0.0001)
conf_options_parser.add_argument("--max-number",
help="The max number of PR points. Default = 10.",
action="store",
dest="max_number",
type=int,
default=10)
conf_options_parser.add_argument("--max-time",
help="Maximum time (seconds). Default is 3600.",
action="store",
dest="max_time",
type=float,
default=3600)
conf_options_parser.add_argument("--csvPrefix",
help="Input file name prefix. Default is ''",
action="store",
dest="csvPrefix",
type=str,
default="")
conf_options_parser.add_argument("--lambda-parm-name",
help="The name of the lambda parameter in the model. Default is lambdaMult",
action="store",
dest="lambda_parm_name",
type=str,
default="lambdaMult")
conf_options_parser.add_argument("--indicator-var-name",
help="The name of the indicator variable for the chance constraint. The default is delta",
action="store",
dest="indicator_var_name",
type=str,
default="delta")
conf_options_parser.add_argument("--stage-num",
help="The stage number of the CC indicator variable (number, not name). Default is 2",
action="store",
dest="stage_num",
type=int,
default=2)
conf_options_parser.add_argument("--verbosity",
help="verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.",
action="store",
dest="verbosity",
type=int,
default=1)
conf_options_parser.add_argument("--prob-file",
help="file name specifiying probabilities",
action="store",
dest="probFileName",
type=str,
default=None)
# The following needed for solve_ph_code in lagrangeutils
conf_options_parser.add_argument("--solve-with-ph",
help="Perform solves via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_with_ph",
default=False)
################################################################
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
if options.verbose is True:
print("Loading reference model and scenario tree")
scenario_instance_factory = \
ScenarioTreeInstanceFactory(options.model_directory,
options.instance_directory)
full_scenario_tree = \
GenerateScenarioTreeForPH(options,
scenario_instance_factory)
solver_manager = SolverManagerFactory(options.solver_manager_type)
if solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type="+options.solver_manager_type+
" specified in call to PH constructor")
if isinstance(solver_manager,
pyomo.solvers.plugins.smanager.phpyro.SolverManager_PHPyro):
solver_manager.deactivate()
raise ValueError("PHPyro can not be used as the solver manager")
try:
if (scenario_instance_factory is None) or (full_scenario_tree is None):
raise RuntimeError("***ERROR: Failed to initialize the model and/or scenario tree data.")
# load_model gets called again, so lets make sure unarchived directories are used
options.model_directory = scenario_instance_factory._model_filename
options.instance_directory = scenario_instance_factory._scenario_tree_filename
scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)
# create ph objects for finding the solution. we do this even if
# we're solving the extensive form
if options.verbose is True:
print("Loading scenario instances and initializing scenario tree for full problem.")
########## Here is where multiplier search is called ############
Result = LagrangeMorePR()
#####################################################################################
finally:
solver_manager.deactivate()
# delete temporary unarchived directories
scenario_instance_factory.close()
print("\n==================== returned from LagrangeMorePR")
print(str(Result.status))
try:
print("Envelope:")
print(str(PrintPRpoints(Result.PRoptimal)))
print("\nAdded:")
PrintPRpoints(Result.morePR)
except:
print("from run: PrintPRpoints failed")
sys.exit()
# combine tables and sort by probability
if len(Result.morePR) > 0:
PRpoints = copy.deepcopy(Result.PRoptimal)
for lbz in Result.morePR: PRpoints.append(lbz)
print("Combined table of PR points (sorted):")
PRpoints.sort(key=operator.itemgetter(1))
print(str(PrintPRpoints(PRpoints)))
class ScenarioTreeServerPyro(TaskWorker):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
mpi = kwds.pop('mpi', None)
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
% (os.getpid(), socket.gethostname()))
if mpi is not None:
assert len(mpi) == 2
kwds["name"] += "_MPIRank_"+str(mpi[1].rank)
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
TaskWorker.__init__(self, **kwds)
assert hasattr(self, "_bulk_task_collection")
self._bulk_task_collection = True
self._contiguous_task_processing = False
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
self._init_verbose = self._verbose
# A reference to the mpi4py.MPI namespace
self.MPI = None
# The communicator and group associated with all processors
self.mpi_comm_world = None
self.mpi_group_world = None
# The communicator associated with the workers assigned
# to the current current client
self.mpi_comm_workers = None
if mpi is not None:
assert len(mpi) == 2
self.MPI = mpi[0]
self.mpi_comm_world = mpi[1]
self.mpi_group_world = self.mpi_comm_world.Get_group()
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
if self.mpi_comm_workers is not None:
self.mpi_comm_workers.Free()
self.mpi_comm_workers = None
self._verbose = self._init_verbose
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
# The only reason we are go through this much
# effort to deal with the serpent serializer
# is because it is the default in Pyro4.
if using_pyro4 and \
(Pyro4.config.SERIALIZER == 'serpent'):
if six.PY3:
assert type(data) is dict
assert data['encoding'] == 'base64'
data = base64.b64decode(data['data'])
else:
assert type(data) is unicode
data = str(data)
return pickle.dumps(self._process(pickle.loads(data)))
except:
logger.error(
"Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle."
% (self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return pickle.dumps(TaskProcessingError(traceback.format_exc()))
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group",None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self,
worker_name,
*data.init_args,
**data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group",None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self,
worker_name,
*data.init_args,
**data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
def __init__(self, fsfile,
fsfct = None,
tree_model = None,
phopts = None):
"""Initialize a StochSolver object.
"""
if fsfct is None:
# Changed in October 2018: None implies AbstractModel
args_list = _optiondict_2_list(phopts)
parser = phinit.construct_ph_options_parser("")
options = parser.parse_args(args_list)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(fsfile, tree_model)
try:
self.scenario_tree = \
phinit.GenerateScenarioTreeForPH(options,
scenario_instance_factory)
except:
print ("ERROR in StochSolver called from",inspect.stack()[1][3])
raise RuntimeError("fsfct is None, so assuming",
"AbstractModel but could not find all ingredients.")
else: # concrete model
if callable(fsfct):
scen_function = fsfct
else: # better be a string
fsfile = fsfile.replace('.py','') # import does not like .py
# __import__ only gives the top level module
# probably need to be dealing with modules installed via setup.py
m = __import__(fsfile)
for n in fsfile.split(".")[1:]:
m = getattr(m, n)
scen_function = getattr(m, fsfct)
if tree_model is None:
treecbname = "pysp_scenario_tree_model_callback"
tree_maker = getattr(m, treecbname)
tree = tree_maker()
if isinstance(tree, Pyo.ConcreteModel):
tree_model = tree
else:
raise RuntimeError("The tree returned by",treecbname,
"must be a ConcreteModel")
scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model)
else:
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
kwargs = _kwfromphopts(phopts)
self.scenario_tree = \
scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
instances = scenario_instance_factory. \
construct_instances_for_scenario_tree(self.scenario_tree)
self.scenario_tree.linkInInstances(instances)
def run(args=None):
##########################================================#########
# to import plugins
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
import pyomo.solvers.plugins.smanager.pyro
def partialLagrangeParametric(args=None):
print("lagrangeParam begins ")
blanks = " " # used for formatting print statements
class Object(object): pass
Result = Object()
# options used
IndVarName = options.indicator_var_name
CCStageNum = options.stage_num
alphaTol = options.alpha_tol
MaxMorePR = options.MaxMorePR # option to include up to this many PR points above F^* with all delta fixed
outputFilePrefix = options.outputFilePrefix
# We write ScenarioList = name, probability
# PRoptimal = probability, min-cost, [selections]
# PRmore = probability, min-cost, [selections]
# ================ sorted by probability ========================
#
# These can be read to avoid re-computing points
ph = PHFromScratch(options)
Result.ph = ph
rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios
if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
print("We are solving a MINIMIZATION problem.\n")
else:
print("We are solving a MAXIMIZATION problem.\n")
# initialize
ScenarioList = []
lambdaval = 0.
lagrUtil.Set_ParmValue(ph,
options.lambda_parm_name,
lambdaval)
# IMPORTANT: Preprocess the scenario instances
# before fixing variables, otherwise they
# will be preprocessed out of the expressions
# and the output_fixed_variable_bounds option
# will have no effect when we update the
# fixed variable values (and then assume we
# do not need to preprocess again because
# of this option).
ph._preprocess_scenario_instances()
lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0)
for scenario in rootnode._scenarios:
ScenarioList.append((scenario._name,
scenario._probability))
# sorts from min to max probability
ScenarioList.sort(key=operator.itemgetter(1))
with open(outputFilePrefix+'ScenarioList.csv','w') as outFile:
for scenario in ScenarioList:
outFile.write(scenario[0]+ ", " +str(scenario[1])+"\n")
Result.ScenarioList = ScenarioList
print("lambda= "+str(lambdaval)+" ...run begins "+str(len(ScenarioList))+" scenarios")
SolStat, zL = lagrUtil.solve_ph_code(ph, options)
print("\t...ends")
bL = Compute_ExpectationforVariable(ph,
IndVarName,
CCStageNum)
if bL > 0:
print("** bL = "+str(bL)+" > 0")
return Result
print("Initial cost = "+str(zL)+" for bL = "+str(bL))
lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1)
print("lambda= "+str(lambdaval)+" ...run begins")
SolStat, zU = lagrUtil.solve_ph_code(ph, options)
print("\t...ends")
bU = Compute_ExpectationforVariable(ph,
IndVarName,
CCStageNum)
if bU < 1:
print("** bU = "+str(bU)+" < 1")
lagrUtil.FreeAllIndicatorVariables(ph, IndVarName)
Result.lbz = [ [0,bL,zL], [None,bU,zU] ]
Result.selections = [[], ScenarioList]
NumIntervals = 1
print("initial gap = "+str(1-zL/zU)+" \n")
print("End of test; this is only a test.")
return Result
################################
# LagrangeParametric ends here
################################
#### start run ####
AllInOne = False
##########################
# options defined here
##########################
try:
conf_options_parser = construct_ph_options_parser("lagrange [options]")
conf_options_parser.add_argument("--alpha",
help="The alpha level for the chance constraint. Default is 0.05",
action="store",
dest="alpha",
type=float,
default=0.05)
conf_options_parser.add_argument("--alpha-min",
help="The min alpha level for the chance constraint. Default is None",
action="store",
dest="alpha_min",
type=float,
default=None)
conf_options_parser.add_argument("--alpha-max",
help="The alpha level for the chance constraint. Default is None",
action="store",
dest="alpha_max",
type=float,
default=None)
conf_options_parser.add_argument("--min-prob",
help="Tolerance for testing probability > 0. Default is 1e-5",
action="store",
dest="min_prob",
type=float,
default=1e-5)
conf_options_parser.add_argument("--alpha-tol",
help="Tolerance for testing equality to alpha. Default is 1e-5",
action="store",
dest="alpha_tol",
type=float,
default=1e-5)
conf_options_parser.add_argument("--MaxMorePR",
help="Generate up to this many additional PR points after response function. Default is 0",
action="store",
dest="MaxMorePR",
type=int,
default=0)
conf_options_parser.add_argument("--outputFilePrefix",
help="Output file name. Default is ''",
action="store",
dest="outputFilePrefix",
type=str,
default="")
conf_options_parser.add_argument("--stage-num",
help="The stage number of the CC indicator variable (number, not name). Default is 2",
action="store",
dest="stage_num",
type=int,
default=2)
conf_options_parser.add_argument("--lambda-parm-name",
help="The name of the lambda parameter in the model. Default is lambdaMult",
action="store",
dest="lambda_parm_name",
type=str,
default="lambdaMult")
conf_options_parser.add_argument("--indicator-var-name",
help="The name of the indicator variable for the chance constraint. The default is delta",
action="store",
dest="indicator_var_name",
type=str,
default="delta")
conf_options_parser.add_argument("--use-Loane-cuts",
help="Add the Loane cuts if there is a gap. Default is False",
action="store_true",
dest="add_Loane_cuts",
default=False)
conf_options_parser.add_argument("--fofx-var-name",
help="(Loane) The name of the model's auxiliary variable that is constrained to be f(x). Default is fofox",
action="store",
dest="fofx_var_name",
type=str,
default="fofx")
conf_options_parser.add_argument("--solve-with-ph",
help="Perform solves via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_with_ph",
default=False)
conf_options_parser.add_argument("--skip-graph",
help="Do not show the graph at the end. Default is False (i.e. show the graph)",
action="store_true",
dest="skip_graph",
default=False)
conf_options_parser.add_argument("--write-xls",
help="Write results into a xls file. Default is False",
action="store_true",
dest="write_xls",
default=False)
conf_options_parser.add_argument("--skip-ExpFlip",
help="Do not show the results for flipping the indicator variable for each scenario. Default is False (i.e. show the flipping-results)",
action="store_true",
dest="skip_ExpFlip",
default=False)
conf_options_parser.add_argument("--HeurFlip",
help="The number of solutions to evaluate after the heuristic. Default is 3. For 0 the heuristic flip gets skipped.",
action="store",
type=int,
dest="HeurFlip",
default=3)
conf_options_parser.add_argument("--HeurMIP",
help="The mipgap for the scenariowise solves in the heuristic. Default is 0.0001",
action="store",
type=float,
dest="HeurMIP",
default=0.0001)
conf_options_parser.add_argument("--interactive",
help="Enable interactive version of the code. Default is False.",
action="store_true",
dest="interactive",
default=False)
conf_options_parser.add_argument("--Lgap",
help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4",
action="store",
type=float,
dest="LagrangeGap",
default=0.0001)
conf_options_parser.add_argument("--lagrange-method",
help="The Lagrange multiplier search method",
action="store",
dest="lagrange_search_method",
type=str,
default="tangential")
conf_options_parser.add_argument("--max-lambda",
help="The max value of the multiplier. Default=10^10",
action="store",
dest="max_lambda",
type=float,
default=10**10)
conf_options_parser.add_argument("--min-lambda",
help="The min value of the multiplier. Default=0.0",
action="store",
dest="min_lambda",
type=float,
default=0)
conf_options_parser.add_argument("--min-probability",
help="The min value of scenario probability. Default=10^-15",
action="store",
dest="min_probability",
type=float,
default=10**(-15))
################################################################
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
# load the reference model and create the scenario tree - no
# scenario instances yet.
if options.verbose:
print("Loading reference model and scenario tree")
#scenario_instance_factory, full_scenario_tree = load_models(options)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(options.model_directory,
options.instance_directory)
full_scenario_tree = \
GenerateScenarioTreeForPH(options,
scenario_instance_factory)
solver_manager = SolverManagerFactory(options.solver_manager_type)
if solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type="+options.solver_manager_type+
" specified in call to PH constructor")
if isinstance(solver_manager,
pyomo.solvers.plugins.smanager.phpyro.SolverManager_PHPyro):
raise ValueError("PHPyro can not be used as the solver manager")
try:
if (scenario_instance_factory is None) or (full_scenario_tree is None):
raise RuntimeError("***ERROR: Failed to initialize model and/or the scenario tree data.")
# load_model gets called again, so lets make sure unarchived directories are used
options.model_directory = scenario_instance_factory._model_filename
options.instance_directory = scenario_instance_factory._scenario_tree_filename
scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)
# create ph objects for finding the solution. we do this even if
# we're solving the extensive form
if options.verbose:
print("Loading scenario instances and initializing scenario tree for full problem.")
########## Here is where multiplier search is called ############
Result = partialLagrangeParametric()
#####################################################################################
finally:
# delete temporary unarchived directories
scenario_instance_factory.close()
print("\nreturned from partialLagrangeParametric")
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
mpi = kwds.pop('mpi', None)
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" %
(os.getpid(), socket.gethostname()))
if mpi is not None:
assert len(mpi) == 2
kwds["name"] += "_MPIRank_" + str(mpi[1].rank)
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
pyu_pyro.TaskWorker.__init__(self, **kwds)
assert hasattr(self, "_bulk_task_collection")
self._bulk_task_collection = True
self._contiguous_task_processing = False
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
self._init_verbose = self._verbose
# A reference to the mpi4py.MPI namespace
self.MPI = None
# The communicator and group associated with all processors
self.mpi_comm_world = None
self.mpi_group_world = None
# The communicator associated with the workers assigned
# to the current current client
self.mpi_comm_workers = None
if mpi is not None:
assert len(mpi) == 2
self.MPI = mpi[0]
self.mpi_comm_world = mpi[1]
self.mpi_group_world = self.mpi_comm_world.Get_group()
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
if self.mpi_comm_workers is not None:
self.mpi_comm_workers.Free()
self.mpi_comm_workers = None
self._verbose = self._init_verbose
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
# The only reason we are go through this much
# effort to deal with the serpent serializer
# is because it is the default in Pyro4.
if pyu_pyro.using_pyro4 and \
(Pyro4.config.SERIALIZER == 'serpent'):
if six.PY3:
assert type(data) is dict
assert data['encoding'] == 'base64'
data = base64.b64decode(data['data'])
else:
assert type(data) is unicode
data = str(data)
return pickle.dumps(self._process(pickle.loads(data)))
except:
logger.error("Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle." %
(self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return pickle.dumps(
pyu_pyro.TaskProcessingError(traceback.format_exc()))
def _process(self, data):
data = Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group", None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request." % (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self, worker_name, *data.init_args, **data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result
def _process(self, data):
data = Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group", None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request." % (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self, worker_name, *data.init_args, **data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result
