def test_random_bundles(self):
     self.assertTrue("ReferenceModel" not in sys.modules)
     scenario_instance_factory = \
         ScenarioTreeInstanceFactory(farmer_model_dir,
                                     farmer_scenariotree_dir)
     scenario_tree = \
         scenario_instance_factory.generate_scenario_tree(random_bundles=2)
     self.assertEqual(scenario_tree.contains_bundles(), True)
     self.assertEqual(len(scenario_tree._scenario_bundles), 2)
     self.assertTrue("ReferenceModel" in sys.modules)
     del sys.modules["ReferenceModel"]
Exemple #2
0
def run(args=None):
###################################

   print("RUNNING - run args=%s" % str(args))

   import pyomo.environ

   def LagrangeParametric(args=None):
      class Object(object): pass
      Result = Object()
      Result.status = 'LagrangeParam begins '+ datetime_string() + '...running new ph'
      ph = None

      blanks = "                          "  # used for formatting print statements
# options used
      betaMin       = options.beta_min
      betaMax       = options.beta_max
      betaTol       = options.beta_tol
      gapTol        = options.Lagrange_gap
      minProb       = options.min_prob
      maxIntervals  = options.max_intervals
      maxTime       = options.max_time
      IndVarName    = options.indicator_var_name
      multName      = options.lambda_parm_name
      CCStageNum    = options.stage_num
      csvPrefix     = options.csvPrefix
      verbosity     = options.verbosity
      verbosity = 2 # override for debug (= 3 to get super-debug)
      HGdebug = 0   # special debug (not public)
# local...may become option
      optTol = gapTol
####################################################################
      STARTTIME = time.time()

      Result.status = "options set"
      if verbosity > 1:
        print("From LagrangeParametric, status = %s\tSTARTTIME = %s" \
                % (str(getattr(Result,'status')), str(STARTTIME)))

      ph = PHFromScratch(options)
      Result.ph = ph
      rootnode = ph._scenario_tree._stages[0]._tree_nodes[0]   # use rootnode to loop over scenarios
      ReferenceInstance = ph._instances[rootnode._scenarios[0]._name]  # arbitrary scenario

      if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
         sense = 'min'
      else:
         sense = 'max'

      scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)
      if options.verbosity > 0: print("%s %s scenarios" % (str(sense),str(scenario_count)))

# initialize
      Result.status = 'starting at '+datetime_string()
      if verbosity > 0:
         print(Result.status)
      ScenarioList = []
      lambdaval = 0.
      lagrUtil.Set_ParmValue(ph, multName,lambdaval)

      # IMPORTANT: Preprocess the scenario instances
      #            before fixing variables, otherwise they
      #            will be preprocessed out of the expressions
      #            and the output_fixed_variable_bounds option
      #            will have no effect when we update the
      #            fixed variable values (and then assume we
      #            do not need to preprocess again because
      #            of this option).
      ph._preprocess_scenario_instances()

      sumprob = 0.
      minprob = 1.
      maxprob = 0.
      # fixed = 0 to get PR point at b=0
      lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0)
      for scenario in rootnode._scenarios:
         instance = ph._instances[scenario._name]
         sname = scenario._name
         sprob = scenario._probability
         sumprob = sumprob + sprob
         minprob = min(minprob,sprob)
         maxprob = max(maxprob,sprob)
         ScenarioList.append([sname,sprob])

      ScenarioList.sort(key=operator.itemgetter(1))   # sorts from min to max probability
      if verbosity > 0:
         print("probabilities sum to %f range: %f to %f" % (sumprob,minprob,maxprob))
      Result.ScenarioList = ScenarioList

# Write ScenarioList = name, probability in csv file sorted by probability
      outName = csvPrefix + 'ScenarioList.csv'
      print("writing to %s" % outName)
      with open(outName,'w') as outFile:
         for scenario in ScenarioList:
            outFile.write(scenario[0]+", "+str(scenario[1])+'\n')
      Result.ScenarioList = ScenarioList

      addstatus = 'Scenario List written to ' + csvPrefix+'ScenarioList.csv'
      Result.status = Result.status + '\n' + addstatus
      if verbosity > 0:
         print(addstatus)

      if verbosity > 0:
         print("solve begins %s" % datetime_string())
         print("\t- lambda = %f" % lambdaval)
      SolStat, zL = lagrUtil.solve_ph_code(ph, options)
      if verbosity > 0:
         print("solve ends %s" % datetime_string())
         print("\t- status = %s" % str(SolStat))
         print("\t- zL = %s" % str(zL))

      bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
      if bL > 0:
         print("** bL = %s > 0 (all %s = 0)" % (str(bL), str(IndVarName)))
         return Result

      if verbosity > 0:  print("Initial optimal obj = %s for bL = %s" % (str(zL), str(bL)))

      # fixed = 1 to get PR point at b=1
      lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1)

      if verbosity > 0:
        print("solve begins %s" % datetime_string())
        print("\t- lambda = %s" % str(lambdaval))
      SolStat, zU = lagrUtil.solve_ph_code(ph, options)
      if verbosity > 0:
        print("solve ends %s" % datetime_string())
        print("\t- status = %s" % str(SolStat))
        print("\t- zU = %s" % str(zU))
      if not SolStat[0:2] == 'ok':
         print(str(SolStat[0:3])+" is not 'ok'")
         addstatus = "** Solution is non-optimal...aborting"
         print(addstatus)
         Result.status = Result.status + "\n" + addstatus
         return Result

      bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
      if bU < 1.- betaTol and verbosity > 0:
         print("** Warning:  bU = %s  < 1" % str(bU))

### enumerate points in PR space (all but one scenario)
#      Result.lbz = [ [0,bL,zL], [None,bU,zU] ]
#      for scenario in rootnode._scenarios:
#         sname = scenario._name
#         instance = ph._instances[sname]
#         print "excluding scenario",sname
#         getattr(instance,IndVarName).value = 0
#         print sname,"value =",getattr(instance,IndVarName).value,getattr(instance,IndVarName).fixed
#         SolStat, z = lagrUtil.solve_ph_code(ph, options)
#         b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
#         print "solve ends with status =",SolStat,"(b, z) =",b,z
#         getattr(instance,IndVarName).value = 1
#         Result.lbz.append([None,b,z])
#         for t in instance.TimePeriods:
#           print "Global at",t,"=",instance.posGlobalLoadGenerateMismatch[t].value, \
#                '-',instance.negGlobalLoadGenerateMismatch[t].value,"=",\
#                    instance.GlobalLoadGenerateMismatch[t].value,\
#               "\tDemand =",instance.TotalDemand[t].value, ",",\
#                "Reserve =",instance.ReserveRequirement[t].value
#
#      PrintPRpoints(Result.lbz)
#      return Result
#### end enumeration
########################################################################

      if verbosity > 1:
         print("We have bU = %s ...about to free all %s for %d scenarios" % \
                (str(bU), str(IndVarName), len(ScenarioList)))

      # free scenario selection variable
      lagrUtil.FreeAllIndicatorVariables(ph, IndVarName)

      if verbosity > 1:
         print("\tall %s freed; elapsed time = %f" % (str(IndVarName), time.time() - STARTTIME))

# initialize with the two endpoints
      Result.lbz = [ [0.,bL,zL], [None,bU,zU] ]
      Result.selections = [[], ScenarioList]
      NumIntervals = 1
      if verbosity > 0:
         print("Initial relative Lagrangian gap = %f maxIntervals = %d" % (1-zL/zU, maxIntervals))
         if verbosity > 1:
            print("entering while loop %s" % datetime_string())
         print("\n")

############ main loop to search intervals #############
########################################################
      while NumIntervals < maxIntervals:
         lapsedTime = time.time() - STARTTIME
         if lapsedTime > maxTime:
            addstatus = '** max time reached ' + str(lapsedTime)
            print(addstatus)
            Result.status = Result.status + '\n' + addstatus
            break
         if verbosity > 1:
            print("Top of while with %d intervals elapsed time = %f" % (NumIntervals, lapsedTime))
            PrintPRpoints(Result.lbz)

         lambdaval = None
### loop over PR points to find first unfathomed interval to search ###
         for PRpoint in range(1,len(Result.lbz)):
            if Result.lbz[PRpoint][0] == None:
# multiplier = None means interval with upper endpoint at PRpoint not fathomed
               bL = Result.lbz[PRpoint-1][1]
               zL = Result.lbz[PRpoint-1][2]
               bU = Result.lbz[PRpoint][1]
               zU = Result.lbz[PRpoint][2]
               lambdaval = (zU - zL) / (bU - bL)
               break

#############################
# Exited from the for loop
         if verbosity > 1:
            print("exited for loop with PRpoint = %s ...lambdaval = %s" % (PRpoint, lambdaval))
         if lambdaval == None: break # all intervals are fathomed

         if verbosity > 1: PrintPRpoints(Result.lbz)
         if verbosity > 0:
            print("Searching for b in [%s, %s] with %s = %f" % (str(round(bL,4)), str(round(bU,4)), multName, lambdaval))

# search interval (bL,bU)
         lagrUtil.Set_ParmValue(ph, multName,lambdaval)
         if verbosity > 0:
            print("solve begins %s" % datetime_string())
            print("\t- %s = %f" % (multName, lambdaval))

         #########################################################
         SolStat, Lagrangian = lagrUtil.solve_ph_code(ph, options)
         #########################################################
         if not SolStat[0:2] == 'ok':
            addstatus = "** Solution status " + SolStat + " is not optimal"
            print(addstatus)
            Result.status = Result.status + "\n" + addstatus
            return Result

         b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
         z = Lagrangian + lambdaval*b
         if verbosity > 0:
            print("solve ends %s" % datetime_string())
            print("\t- Lagrangian = %f" % Lagrangian)
            print("\t- b = %s" % str(b))
            print("\t- z = %s" % str(z))
            print("\n")

# We have PR point (b,z), which may be new or one of the endpoints
##################################################################

######### Begin tolerance tests ##########
# Test that b is in [bL,bU]
         if verbosity > 1: print("\ttesting b")
         if b < bL - betaTol or b > bU + betaTol:
            addstatus = "** fatal error: probability (= " + str(b) + \
                ") is outside interval, (" + str(bL) + ", " + str(bU) + ")"
            addstatus = addstatus + "\n\t(tolerance = " + str(betaTol) + ")"
            print(addstatus+'\n')
            Result.status = Result.status + addstatus
            return Result
# Test that z is in [zL,zU]
         if verbosity > 1: print("\ttesting z")
# using optTol as absolute tolerance (not relative)
#   ...if we reconsider, need to allow negative z-values
         if z < zL - optTol or z > zU + optTol:
            addstatus = "** fatal error: obj (= " + str(z) + \
                ") is outside interval, (" + str(zL) + ", " + str(zU) + ")"
            print(addstatus+'\n')
            Result.status = Result.status + addstatus
            return Result

# Ok, we have (b,z) in [(bL,zL), (bU,zU)], at least within tolerances

         oldLagrangian = zL - lambdaval*bL
# ensure lambdaval set such that endpoints have same Lagrangian value
# (this is probably unnecessary, but check anyway)
         if abs(oldLagrangian - (zU - lambdaval*bU)) > optTol*abs(oldLagrangian):
            addstatus = "** fatal error: Lagrangian at (bL,zL) = " + \
                str(oldLagrangian) + " not= " + str(zU-lambdaval*bU) + \
                "\n\t(optTol = " + str(optTol) + ")"
            Result.status = Result.status + addstatus
            return Result

# no more fatal error tests...need to know if (b,z) is endpoint or new

         if verbosity > 1: print("No anomalies...testing if b = bL or bU")

# Test if endpoint is an alternative optimum of Lagrangian
# ...using optTol as *relative* tolerance
# (could use other reference values -- eg, avg or max of old and new Lagrangian values)
         refValue =  max( min( abs(oldLagrangian), abs(Lagrangian) ), 1.)
         alternativeOpt = abs( oldLagrangian - Lagrangian ) <= optTol*refValue

# alternativeOpt = True means we computed point (b,z) is alternative optimum such that:
#   case 1: (b,z) = endpoint, in which case we simply fathom [bL,bU] by setting PRpoint
#            to [lambdaval,bU,zU] (the numeric value of multiplier means fathomed)
#   case 2: (b,z) is new PR point on line segment, in which case we split into
#           [bL,b] and [b,bU], with both fathomed

         if verbosity > 1:
            print("oldLagrangian = %s" % str(oldLagrangian))
            if alternativeOpt: print(":= Lagrangian = %s" % str(Lagrangian ))
            else: print("> Lagrangian = %s" % str(Lagrangian))

         if alternativeOpt:
# setting multiplier of (bU,zU) to a numeric fathoms the interval [bL,bU]
            Result.lbz[PRpoint][0] = lambdaval

# test if (b,z) is an endpoint
         newPRpoint = abs(b-bL) > betaTol and abs(b-bU) > betaTol
         if not newPRpoint:
# ...(b,z) is NOT an endpoint (or sufficiently close), so split and fathom
            if verbosity > 1:
               print("\tnot an endpoint\tlbz = %s" % str(Result.lbz[PRpoint]))
            if verbosity > 0:
               print("Lagangian solution is new PR point on line segment of (" \
                  + str(bL) + ", " + str(bU) +")")
               print("\tsplitting (bL,bU) into (bL,b) and (b,bU), both fathomed")
# note:  else ==> b = bL or bU, so we do nothing, having already fathomed [bL,bU]

# (b,z) is new PR point, so split interval (still in while loop)
##########################################
# alternative optimum ==> split & fathom: (bL,b), (b,bU)
         if verbosity > 1:
            print("\talternativeOpt %s newPRpoint = %s" % (alternativeOpt, newPRpoint))
         if newPRpoint:
            NumIntervals += 1
            if alternativeOpt:
               if verbosity > 1: print("\tInsert [lambdaval,b,z] at %f" % PRpoint)
               Result.lbz = Insert([lambdaval,b,z],PRpoint,Result.lbz)
               addstatus = "Added PR point on line segment of envelope"
               if verbosity > 0: print(addstatus+'\n')
            else:
               if verbosity > 1: print("\tInsert [None,b,z] at %f" % PRpoint)
               Result.lbz = Insert([None,b,z],PRpoint,Result.lbz)
               addstatus = "new envelope extreme point added (interval split, not fathomed)"
            Result.status = Result.status + "\n" + addstatus

            if verbosity > 1:
               print("...after insertion:")
               PrintPRpoints(Result.lbz)

# get the selections of new point (ie, scenarios for which delta=1)
            Selections = []
            for scenario in ScenarioList:
               instance = ph._instances[scenario[0]]
               if getattr(instance,IndVarName).value == 1:
                  Selections.append(scenario)
            Result.selections = Insert(Selections,PRpoint,Result.selections)

            if verbosity > 0:
               print("Interval "+str(PRpoint)+", ["+str(bL)+", "+str(bU)+ \
                 "] split at ("+str(b)+", "+str(z)+")")
               print("\tnew PR point has "+str(len(Selections))+" selections")

            if verbosity > 1: print("test that selections list aligned with lbz")
            if not len(Result.lbz) == len(Result.selections):
               print("** fatal error: lbz not= selections")
               PrintPRpoints(Result.lbz)
               print("Result.selections:")
               for i in range(Result.selections): print("%d %f" % (i,Result.selections[i]))
               return Result

# ok, we have split and/or fathomed interval
         if NumIntervals >= maxIntervals:
# we are about to leave while loop due to...
            addstatus = "** terminating because number of intervals = " + \
                    str(NumIntervals) + " >= max = " + str(maxIntervals)
            if verbosity > 0: print(addstatus+'\n')
            Result.status = Result.status + "\n" + addstatus

# while loop continues
         if verbosity > 1:
            print("bottom of while loop")
            PrintPRpoints(Result.lbz)

###################################################
# end while NumIntervals < maxIntervals:
#     ^ this is indentation of while loop
################ end while loop ###################

      if verbosity > 1:  print("\nend while loop...setting multipliers")
      for i in range(1,len(Result.lbz)):
         db = Result.lbz[i][1] - Result.lbz[i-1][1]
         dz = Result.lbz[i][2] - Result.lbz[i-1][2]
         if dz > 0:
            Result.lbz[i][0] = dz/db
         else:
            #print "dz =",dz," at ",i,": ",Result.lbz[i]," -",Result.lbz[i-1]
            Result.lbz[i][0] = 0
      if verbosity > 0: PrintPRpoints(Result.lbz)

      addstatus = '\nLagrange multiplier search ends'+datetime_string()
      if verbosity > 0:
         print(addstatus+'\n')
      Result.status = Result.status + addstatus

      outName = csvPrefix + "PRoptimal.csv"
      with open(outName,'w') as outFile:
         if verbosity > 0:
            print("writing PR points to "+outName+'\n')
         for lbz in Result.lbz:
            outFile.write(str(lbz[1])+ ", " +str(lbz[2])+'\n')

      outName = csvPrefix + "OptimalSelections.csv"
      with open(outName,'w') as outFile:
         if verbosity > 0:
            print("writing optimal selections for each PR point to "+csvPrefix+'PRoptimal.csv\n')
         for selections in Result.selections:
            char = ""
            thisSelection = ""
            for slist in selections:
               if slist:
                  thisSelection = thisSelection + char + slist[0]
                  char = ","
            outFile.write(thisSelection+'\n')

      if verbosity > 0:
         print("\nReturning status:\n %s \n=======================" % Result.status)

################################
      if verbosity > 2:
         print("\nAbout to return...Result attributes: %d" % len(inspect.getmembers(Result)))
         for attr in inspect.getmembers(Result): print(attr[0])
         print("\n===========================================")
# LagrangeParametric ends here
      return Result
################################


####################################### start run ####################################

   AllInOne = False

########################
# options defined here
########################
   try:
      conf_options_parser = construct_ph_options_parser("lagrange [options]")
      conf_options_parser.add_argument("--beta-min",
                                     help="The min beta level for the chance constraint. Default is 0",
                                     action="store",
                                     dest="beta_min",
                                     type=float,
                                     default=0.)
      conf_options_parser.add_argument("--beta-max",
                                     help="The beta level for the chance constraint. Default is 1.",
                                     action="store",
                                     dest="beta_max",
                                     type=float,
                                     default=1.)
      conf_options_parser.add_argument("--beta-tol",
                                     help="Tolerance for testing equality to beta. Default is 1e-5",
                                     action="store",
                                     dest="beta_tol",
                                     type=float,
                                     default=1e-5)
      conf_options_parser.add_argument("--Lagrange-gap",
                                     help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4",
                                     action="store",
                                     type=float,
                                     dest="Lagrange_gap",
                                     default=0.0001)
      conf_options_parser.add_argument("--min-prob",
                                     help="Tolerance for testing probability > 0. Default is 1e-9",
                                     action="store",
                                     dest="min_prob",
                                     type=float,
                                     default=1e-5)
      conf_options_parser.add_argument("--max-intervals",
                                     help="The max number of intervals generated; if causes termination, non-fathomed intervals have multiplier=None.  Default = 100.",
                                     action="store",
                                     dest="max_intervals",
                                     type=int,
                                     default=100)
      conf_options_parser.add_argument("--max-time",
                                     help="Maximum time (seconds). Default is 3600.",
                                     action="store",
                                     dest="max_time",
                                     type=float,
                                     default=3600)
      conf_options_parser.add_argument("--lambda-parm-name",
                                     help="The name of the lambda parameter in the model. Default is lambdaMult",
                                     action="store",
                                     dest="lambda_parm_name",
                                     type=str,
                                     default="lambdaMult")
      conf_options_parser.add_argument("--indicator-var-name",
                                     help="The name of the indicator variable for the chance constraint. The default is delta",
                                     action="store",
                                     dest="indicator_var_name",
                                     type=str,
                                     default="delta")
      conf_options_parser.add_argument("--stage-num",
                                     help="The stage number of the CC indicator variable (number, not name). Default is 2",
                                     action="store",
                                     dest="stage_num",
                                     type=int,
                                     default=2)
      conf_options_parser.add_argument("--csvPrefix",
                                     help="Output file name.  Default is ''",
                                     action="store",
                                     dest="csvPrefix",
                                     type=str,
                                     default='')
      conf_options_parser.add_argument("--verbosity",
                                     help="verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.",
                                     action="store",
                                     dest="verbosity",
                                     type=int,
                                     default=1)
# The following needed for solve_ph_code in lagrangeutils
      conf_options_parser.add_argument("--solve-with-ph",
                                     help="Perform solves via PH rather than an EF solve. Default is False",
                                     action="store_true",
                                     dest="solve_with_ph",
                                     default=False)
##HG: deleted params filed as deletedParam.py
#######################################################################################################

      options = conf_options_parser.parse_args(args=args)
      # temporary hack
      options._ef_options = conf_options_parser._ef_options
      options._ef_options.import_argparse(options)
   except SystemExit as _exc:
      # the parser throws a system exit if "-h" is specified - catch
      # it to exit gracefully.
      return _exc.code

   # create the reference instances and the scenario tree - no
   # scenario instances yet.
   if options.verbosity > 0:
        print("Loading reference model and scenario tree")
# Dec 18
#   scenario_instance_factory, full_scenario_tree = load_models(options)
   scenario_instance_factory = \
        ScenarioTreeInstanceFactory(options.model_directory,
                                    options.instance_directory)

   full_scenario_tree = \
            GenerateScenarioTreeForPH(options,
                                      scenario_instance_factory)

####
   try:
      if (scenario_instance_factory is None) or (full_scenario_tree is None):
         raise RuntimeError("***ERROR: Failed to initialize the model and/or scenario tree data.")

      # load_model gets called again, so lets make sure unarchived directories are used
      options.model_directory = scenario_instance_factory._model_filename
      options.instance_directory = scenario_instance_factory._scenario_tree_filename

########## Here is where multiplier search is called from run() ############
      Result = LagrangeParametric()
#####################################################################################
   finally:

      # delete temporary unarchived directories
      scenario_instance_factory.close()

   if options.verbosity > 0:
      print("\n===========================================")
      print("\nreturned from LagrangeParametric")
      if options.verbosity > 2:
         print("\nFrom run, Result should have status and ph objects...")
         for attr in inspect.getmembers(Result): print(attr)
         print("\n===========================================")

   try:
     status = Result.status
     print("status = "+str(Result.status))
   except:
     print("status not defined")
     sys.exit()

   try:
      lbz = Result.lbz
      PrintPRpoints(lbz)
      with open(options.csvPrefix+"PRoptimal.csv",'w') as outFile:
         for lbz in Result.lbz:
            outFile.write(str(lbz[1])+ ", " +str(lbz[2])+'\n')
   except:
      print("Result.lbz not defined")
      sys.exit()

   try:
      ScenarioList = Result.ScenarioList
      ScenarioList.sort(key=operator.itemgetter(1))
      with open(options.csvPrefix+"ScenarioList.csv",'w') as outFile:
         for scenario in ScenarioList:
            outFile.write(scenario[0]+", "+str(scenario[1])+'\n')
   except:
      print("Result.ScenarioList not defined")
      sys.exit()
Exemple #3
0
def run(args=None):
    AllInOne = False
    # The value of AllInOne will be set to True for the "old"
    # computeconf (with fraction_for_solve) and will stay False for
    # the "new" computeconf (with MRP_directory_basename)

    try:
        conf_options_parser = construct_ph_options_parser("computeconf [options]")
        conf_options_parser.add_argument(
            "--fraction-scenarios-for-solve",
            help="The fraction of scenarios that are allocated to finding a solution. Default is None.",
            action="store",
            dest="fraction_for_solve",
            type=float,
            default=None,
        )
        conf_options_parser.add_argument(
            "--number-samples-for-confidence-interval",
            help="The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.",
            action="store",
            dest="n_g",
            type=int,
            default=None,
        )
        conf_options_parser.add_argument(
            "--confidence-interval-alpha",
            help="The alpha level for the confidence interval. Default is 0.05",
            action="store",
            dest="confidence_interval_alpha",
            type=float,
            default=0.05,
        )
        conf_options_parser.add_argument(
            "--solve-xhat-with-ph",
            help="Perform xhat solve via PH rather than an EF solve. Default is False",
            action="store_true",
            dest="solve_xhat_with_ph",
            default=False,
        )
        conf_options_parser.add_argument(
            "--random-seed",
            help="Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.",
            action="store",
            dest="random_seed",
            type=int,
            default=0,
        )
        conf_options_parser.add_argument(
            "--append-file",
            help="File to which summary run information is appended, for output tracking purposes.",
            action="store",
            dest="append_file",
            type=str,
            default=None,
        )
        conf_options_parser.add_argument(
            "--write-xhat-solution",
            help="Write xhat solutions (first stage variables only) to the append file? Defaults to False.",
            action="store_true",
            dest="write_xhat_solution",
            default=False,
        )
        conf_options_parser.add_argument(
            "--generate-weighted-cvar",
            help="Add a weighted CVaR term to the primary objective",
            action="store_true",
            dest="generate_weighted_cvar",
            default=False,
        )
        conf_options_parser.add_argument(
            "--cvar-weight",
            help="The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.",
            action="store",
            dest="cvar_weight",
            type=float,
            default=1.0,
        )
        conf_options_parser.add_argument(
            "--risk-alpha",
            help="The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.",
            action="store",
            dest="risk_alpha",
            type=float,
            default=0.95,
        )
        conf_options_parser.add_argument(
            "--MRP-directory-basename",
            help="The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None",
            action="store",
            dest="MRP_directory_basename",
            type=str,
            default=None,
        )

        options = conf_options_parser.parse_args(args=args)
        # temporary hack
        options._ef_options = conf_options_parser._ef_options
        options._ef_options.import_argparse(options)
    except SystemExit as _exc:
        # the parser throws a system exit if "-h" is specified - catch
        # it to exit gracefully.
        return _exc.code

    # seed the generator if a user-supplied seed is
    # provided. otherwise, python will seed from the current system
    # time.
    if options.random_seed > 0:
        random.seed(options.random_seed)

    start_time = time.time()
    if options.verbose:
        print("Importing model and scenario tree files")

    scenario_instance_factory = ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory)
    if _OLD_OUTPUT:
        print("Loading reference model and scenario tree")
    if options.verbose or options.output_times:
        print("Time to import model and scenario " "tree structure files=%.2f seconds" % (time.time() - start_time))

    try:

        scenario_tree = scenario_instance_factory.generate_scenario_tree(
            downsample_fraction=options.scenario_tree_downsample_fraction,
            bundles=options.scenario_bundle_specification,
            random_bundles=options.create_random_bundles,
            random_seed=options.scenario_tree_random_seed,
            verbose=options.verbose,
        )

        #
        # print the input tree for validation/information purposes.
        #
        if options.verbose:
            scenario_tree.pprint()

        #
        # validate the tree prior to doing anything serious
        #
        if not scenario_tree.validate():
            raise RuntimeError("Scenario tree is invalid")
        else:
            if options.verbose:
                print("Scenario tree is valid!")

        index_list, num_scenarios_for_solution, num_scenarios_per_sample = partition_scenario_space(
            scenario_tree, options
        )

        # index_list = [0,3,5,7,1,4,6,8,2,9]
        # for ndx in index_list:
        #    print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name))
        xhat_ph = find_candidate(
            scenario_instance_factory, index_list, num_scenarios_for_solution, scenario_tree, options
        )

        run_conf(
            scenario_instance_factory,
            index_list,
            num_scenarios_for_solution,
            num_scenarios_per_sample,
            scenario_tree,
            xhat_ph,
            options,
        )

    finally:

        # delete temporary unarchived directories
        if scenario_instance_factory is not None:
            scenario_instance_factory.close()
Exemple #4
0
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject):

    # Maps name to a registered worker class to instantiate
    _registered_workers = {}

    _declared_options = \
        PySPConfigBlock("Options declared for the "
                        "ScenarioTreeServerPyro class")

    #
    # scenario instance construction
    #
    safe_declare_common_option(_declared_options,
                               "model_location")
    safe_declare_common_option(_declared_options,
                               "scenario_tree_location")

    #
    # scenario tree generation
    #
    safe_declare_common_option(_declared_options,
                               "scenario_tree_random_seed")
    safe_declare_common_option(_declared_options,
                               "scenario_tree_downsample_fraction")

    #
    # various
    #
    safe_declare_common_option(_declared_options,
                               "verbose")

    @classmethod
    def get_registered_worker_type(cls, name):
        if name in cls._registered_workers:
            return cls._registered_workers[name]
        raise KeyError("No worker type has been registered under the name "
                       "'%s' for ScenarioTreeServerPyro" % (name))

    def __init__(self, *args, **kwds):


        # add for purposes of diagnostic output.
        kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
                        % (os.getpid(), socket.gethostname()))
        kwds["caller_name"] = kwds["name"]
        self._modules_imported = kwds.pop('modules_imported', {})

        TaskWorker.__init__(self, **kwds)
        # This classes options get updated during the "setup" phase
        options = self.register_options()
        PySPConfiguredObject.__init__(self, options)

        self.type = self.WORKERNAME
        self.block = True
        self.timeout = None
        self._worker_map = {}

        #
        # These will be used by all subsequent workers created
        # by this server. Their creation can eat up a nontrivial
        # amount of initialization time when a large number of
        # workers are created on this server, so we only create
        # them once.
        #
        self._scenario_instance_factory = None
        self._full_scenario_tree = None

    def reset(self):
        if self._scenario_instance_factory is not None:
            self._scenario_instance_factory.close()
        self._scenario_instance_factory = None
        self._full_scenario_tree = None
        for worker_name in list(self._worker_map):
            self.remove_worker(worker_name)

    def remove_worker(self, name):
        self._worker_map[name].close()
        del self._worker_map[name]

    def process(self, data):
        self._worker_task_return_queue = self._current_task_client
        try:
            return self._process(data)
        except:
            logger.error(
                "Scenario tree server %s caught an exception of type "
                "%s while processing a task. Going idle."
                % (self.WORKERNAME, sys.exc_info()[0].__name__))
            traceback.print_exception(*sys.exc_info())
            self._worker_error = True
            return TaskProcessingError(traceback.format_exc())

    def _process(self, data):

        data = pyutilib.misc.Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            #with PauseGC() as pgc:
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            options = self.register_options()
            for name, val in iteritems(data.options):
                options.get(name).set_value(val)
            self.set_options(options)
            self._options.verbose = self._options.verbose | self._verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._options.verbose:
                print("Server %s received setup request."
                      % (self.WORKERNAME))
                print("Options:")
                self.display_options()

            # Make sure these are not archives
            assert os.path.exists(self._options.model_location)
            assert (self._options.scenario_tree_location is None) or \
                os.path.exists(self._options.scenario_tree_location)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    self._options.model_location,
                    scenario_tree_location=self._options.scenario_tree_location,
                    verbose=self._options.verbose)

            #
            # Try prevent unnecessarily re-imported the model module
            # if other callbacks are in the same location
            #
            self._modules_imported[
                self._scenario_instance_factory._model_location] = \
                    self._scenario_instance_factory._model_module
            self._modules_imported[
                self._scenario_instance_factory._model_filename] = \
                    self._scenario_instance_factory._model_module

            self._full_scenario_tree = \
                self._scenario_instance_factory.generate_scenario_tree(
                    downsample_fraction=self._options.scenario_tree_downsample_fraction,
                    random_seed=self._options.scenario_tree_random_seed)

            if self._full_scenario_tree is None:
                 raise RuntimeError("Unable to launch scenario tree worker - "
                                    "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._options.verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s."
                      % (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name."
                     % (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]
            options = worker_type.register_options()
            for name, val in iteritems(data.options):
                options.get(name).set_value(val)

            #
            # Depending on the Pyro serializer, the namedtuple
            # may be been converted to a tuple
            #
            if not isinstance(data.worker_init, WorkerInit):
                assert type(data.worker_init) is tuple
                data.worker_init = WorkerInit(type_=data.worker_init[0],
                                              names=data.worker_init[1],
                                              data=data.worker_init[2])

            # replace enum string representation with the actual enum
            # object now that we've unserialized the Pyro data
            worker_init = WorkerInit(type_=getattr(WorkerInitType,
                                                   data.worker_init.type_),
                                     names=data.worker_init.names,
                                     data=data.worker_init.data)
            self._worker_map[worker_name] = worker_type(options)
            self._worker_map[worker_name].initialize(
                self.WORKERNAME,
                self._full_scenario_tree,
                worker_name,
                worker_init,
                self._modules_imported)

            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._options.verbose:
                print("Server %s releasing worker: %s"
                      % (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._options.verbose:
                print("Server %s received reset request"
                      % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._options.verbose:
                print("Server %s received shutdown request"
                      % (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s"
                             % (self.WORKERNAME, data.action))

        return result
Exemple #5
0
    def _process(self, data):

        data = pyutilib.misc.Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            #with PauseGC() as pgc:
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            options = self.register_options()
            for name, val in iteritems(data.options):
                options.get(name).set_value(val)
            self.set_options(options)
            self._options.verbose = self._options.verbose | self._verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._options.verbose:
                print("Server %s received setup request."
                      % (self.WORKERNAME))
                print("Options:")
                self.display_options()

            # Make sure these are not archives
            assert os.path.exists(self._options.model_location)
            assert (self._options.scenario_tree_location is None) or \
                os.path.exists(self._options.scenario_tree_location)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    self._options.model_location,
                    scenario_tree_location=self._options.scenario_tree_location,
                    verbose=self._options.verbose)

            #
            # Try prevent unnecessarily re-imported the model module
            # if other callbacks are in the same location
            #
            self._modules_imported[
                self._scenario_instance_factory._model_location] = \
                    self._scenario_instance_factory._model_module
            self._modules_imported[
                self._scenario_instance_factory._model_filename] = \
                    self._scenario_instance_factory._model_module

            self._full_scenario_tree = \
                self._scenario_instance_factory.generate_scenario_tree(
                    downsample_fraction=self._options.scenario_tree_downsample_fraction,
                    random_seed=self._options.scenario_tree_random_seed)

            if self._full_scenario_tree is None:
                 raise RuntimeError("Unable to launch scenario tree worker - "
                                    "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._options.verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s."
                      % (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name."
                     % (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]
            options = worker_type.register_options()
            for name, val in iteritems(data.options):
                options.get(name).set_value(val)

            #
            # Depending on the Pyro serializer, the namedtuple
            # may be been converted to a tuple
            #
            if not isinstance(data.worker_init, WorkerInit):
                assert type(data.worker_init) is tuple
                data.worker_init = WorkerInit(type_=data.worker_init[0],
                                              names=data.worker_init[1],
                                              data=data.worker_init[2])

            # replace enum string representation with the actual enum
            # object now that we've unserialized the Pyro data
            worker_init = WorkerInit(type_=getattr(WorkerInitType,
                                                   data.worker_init.type_),
                                     names=data.worker_init.names,
                                     data=data.worker_init.data)
            self._worker_map[worker_name] = worker_type(options)
            self._worker_map[worker_name].initialize(
                self.WORKERNAME,
                self._full_scenario_tree,
                worker_name,
                worker_init,
                self._modules_imported)

            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._options.verbose:
                print("Server %s releasing worker: %s"
                      % (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._options.verbose:
                print("Server %s received reset request"
                      % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._options.verbose:
                print("Server %s received shutdown request"
                      % (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s"
                             % (self.WORKERNAME, data.action))

        return result
def run(args=None):
###################################

   # to import plugins
   import pyomo.environ
   import pyomo.solvers.plugins.smanager.phpyro

   def LagrangeMorePR(args=None):
      print("lagrangeMorePR begins %s" % datetime_string())
      blanks = "                          "  # used for formatting print statements
      class Object(object): pass
      Result = Object()

# options used
      betaTol       = options.beta_tol          # tolerance used to separate b-values
      IndVarName    = options.indicator_var_name
      multName      = options.lambda_parm_name
      CCStageNum    = options.stage_num
      MaxMorePR     = options.max_number         # max PR points to be generated (above F^* with all delta fixed)
      MaxTime       = options.max_time           # max time before terminate
      csvPrefix = options.csvPrefix          # input filename prefix (eg, case name)
      probFileName  = options.probFileName       # name of file containing probabilities
##HG override
#      options.verbosity = 2
      verbosity     = options.verbosity

      Result.status = 'starting '+datetime_string()
      STARTTIME = time.time()

      ph = PHFromScratch(options)
      rootnode = ph._scenario_tree._stages[0]._tree_nodes[0]   # use rootnode to loop over scenarios

      if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
         print("We are solving a MINIMIZATION problem.")
      else:
         print("We are solving a MAXIMIZATION problem.")

# initialize
      ScenarioList = []
      with open(csvPrefix+"ScenarioList.csv",'r') as inputFile:
         for line in inputFile.readlines():
            L = line.split(',')
            ScenarioList.append([L[0],float(L[1])])

      addstatus = str(len(ScenarioList))+' scenarios read from file: ' + csvPrefix+'ScenarioList.csv'
      if verbosity > 0: print(addstatus)
      Result.status = Result.status + '\n' + addstatus

      PRoptimal = []
      with open(csvPrefix+"PRoptimal.csv",'r') as inputFile:
         for line in inputFile.readlines():
            bzS = line.split(',')
            PRoptimal.append( [None, float(bzS[0]), float(bzS[1])] )

      addstatus = str(len(PRoptimal))+' PR points read from file: '+ csvPrefix+'PRoptimal.csv (envelope function)'
      if verbosity > 0:
         print(addstatus)
      Result.status = Result.status + '\n' + addstatus
# ensure PR points on envelope function are sorted by probability
      PRoptimal.sort(key=operator.itemgetter(1))

      PRoptimal[0][0] = 0   # initial lambda (for b=0)
      for p in range(1,len(PRoptimal)):
         dz = PRoptimal[p][2] - PRoptimal[p-1][2]
         db = PRoptimal[p][1] - PRoptimal[p-1][1]
         PRoptimal[p][0] = dz/db
      if verbosity > 0:
         PrintPRpoints(PRoptimal)
      Result.PRoptimal = PRoptimal

      lambdaval = 0.
      lagrUtil.Set_ParmValue(ph, options.lambda_parm_name,lambdaval)

      # IMPORTANT: Preprocess the scenario instances
      #            before fixing variables, otherwise they
      #            will be preprocessed out of the expressions
      #            and the output_fixed_variable_bounds option
      #            will have no effect when we update the
      #            fixed variable values (and then assume we
      #            do not need to preprocess again because
      #            of this option).
      ph._preprocess_scenario_instances()

## read scenarios to select for each PR point on envelope function
      with open(csvPrefix+"OptimalSelections.csv",'r') as inputFile:
         OptimalSelections = []
         for line in inputFile.readlines():
            if len(line) == 0: break # eof
            selections = line.split(',')
            L = len(selections)
            Ls = len(selections[L-1])
            selections[L-1] = selections[L-1][0:Ls-1]
            if verbosity > 1:
               print(str(selections))
            OptimalSelections.append(selections)

      Result.OptimalSelections = OptimalSelections

      addstatus = str(len(OptimalSelections)) + ' Optimal selections read from file: ' \
            + csvPrefix + 'OptimalSelections.csv'
      Result.status = Result.status + '\n' + addstatus

      if len(OptimalSelections) == len(PRoptimal):
         if verbosity > 0:
            print(addstatus)
      else:
         addstatus = addstatus + '\n** Number of selections not equal to number of PR points'
         print(addstatus)
         Result.status = Result.status + '\n' + addstatus
         print(str(OptimalSelections))
         print((PRoptimal))
         return Result

#####################################################################################

# get probabilities
      if probFileName is None:
# ...generate from widest gap regions
         PRlist = FindPRpoints(options, PRoptimal)
      else:
# ...read probabilities
         probList = []
         with open(probFileName,'r') as inputFile:
            if verbosity > 0:
               print("reading from probList = "+probFileName)
            for line in inputFile.readlines():  # 1 probability per line
               if len(line) == 0:
                  break
               prob = float(line)
               probList.append(prob)

         if verbosity > 0:
            print("\t "+str(len(probList))+" probabilities")
         if verbosity > 1:
            print(str(probList))
         PRlist = GetPoints(options, PRoptimal, probList)
         if verbosity > 1:
            print("PRlist:")
            for interval in PRlist:
               print(str(interval))

# We now have PRlist = [[i, b], ...], where b is in PRoptimal interval (i-1,i)
      addstatus = str(len(PRlist)) + ' probabilities'
      if verbosity > 0:
         print(addstatus)
      Result.status = Result.status + '\n' + addstatus

#####################################################################################

      lapsedTime = time.time() - STARTTIME
      addstatus = 'Initialize complete...lapsed time = ' + str(lapsedTime)
      if verbosity > 1:
         print(addstatus)
      Result.status = Result.status + '\n' + addstatus

#####################################################################################

      if verbosity > 1:
        print("\nlooping over Intervals to generate PR points by flipping heuristic")
      Result.morePR = []
      for interval in PRlist:
         lapsedTime = time.time() - STARTTIME
         if lapsedTime > MaxTime:
            addstatus = '** lapsed time = ' + str(lapsedTime) + ' > max time = ' + str(MaxTime)
            if verbosity > 0: print(addstatus)
            Result.status = Result.status + '\n' + addstatus
            break

         i = interval[0] # = PR point index
         b = interval[1] # = target probability to reach by flipping from upper endpoint
         bU = PRoptimal[i][1]   # = upper endpoint
         bL = PRoptimal[i-1][1] # = lower endpoint
         if verbosity > 1:
            print( "target probability = "+str(b)+" < bU = PRoptimal[" + str(i) + "][1]" \
                 " and > bL = PRoptimal["+str(i-1)+"][1]")
         if b < bL or b > bU:
            addstatus = '** probability = '+str(b) + ', not in gap interval: (' \
                + str(bL) + ', ' + str(bU) + ')'
            print(addstatus)
            print(str(PRoptimal))
            print(str(PRlist))
            Result.status = Result.status + '\n' + addstatus
            return Result

         if verbosity > 1:
            print( "i = "+str(i)+" : Starting with bU = "+str(bU)+" having "+ \
                str(len(OptimalSelections[i]))+ " selections:")
            print(str(OptimalSelections[i]))

# first fix all scenarios = 0
         for sname, sprob in ScenarioList:
            scenario = ph._scenario_tree.get_scenario(sname)
            lagrUtil.FixIndicatorVariableOneScenario(ph,
                                                     scenario,
                                                     IndVarName,
                                                     0)

# now fix optimal selections = 1
         for sname in OptimalSelections[i]:
            scenario = ph._scenario_tree.get_scenario(sname)
            lagrUtil.FixIndicatorVariableOneScenario(ph,
                                                     scenario,
                                                     IndVarName,
                                                     1)

# flip scenario selections from bU until we reach b (target probability)
         bNew = bU
         for sname, sprob in ScenarioList:
            scenario = ph._scenario_tree.get_scenario(sname)
            if bNew - sprob < b:
               continue
            instance = ph._instances[sname]
            if getattr(instance, IndVarName).value == 0:
               continue
            bNew = bNew - sprob
            # flipped scenario selection
            lagrUtil.FixIndicatorVariableOneScenario(ph,
                                                     scenario,
                                                     IndVarName,
                                                     0)
            if verbosity > 1:
               print("\tflipped "+sname+" with prob = "+str(sprob)+" ...bNew = "+str(bNew))

         if verbosity > 1:
            print("\tflipped selections reach "+str(bNew)+" >= target = "+str(b)+" (bL = "+str(bL)+")")
         if bNew <= bL + betaTol or bNew >= bU - betaTol:
            if verbosity > 0:
               print("\tNot generating PR point...flipping from bU failed")
            continue # to next interval in list

 # ready to solve to get cost for fixed scenario selections associated with probability = bNew

         if verbosity > 1:
# check that scenarios are fixed as they should be
            totalprob = 0.
            for scenario in ScenarioList:
               sname = scenario[0]
               sprob = scenario[1]
               instance = ph._instances[sname]
               print("fix "+sname+" = "+str(getattr(instance,IndVarName).value)+\
                  " is "+str(getattr(instance,IndVarName).fixed)+" probability = "+str(sprob))
               if getattr(instance,IndVarName).value == 1:
                  totalprob = totalprob + sprob
               lambdaval = getattr(instance, multName).value
            print("\ttotal probability = %f" % totalprob)

# solve (all delta fixed); lambda=0, so z = Lagrangian
         if verbosity > 0:
            print("solve begins %s" % datetime_string())
            print("\t- lambda = %f" % lambdaval)
         SolStat, z = lagrUtil.solve_ph_code(ph, options)
         b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum)
         if verbosity > 0:
            print("solve ends %s" % datetime_string())
            print("\t- SolStat = %s" % str(SolStat))
            print("\t- b = %s" % str(b))
            print("\t- z = %s" % str(z))
            print("(adding to more PR points)")

         Result.morePR.append([None,b,z])
         if verbosity > 1:
            PrintPRpoints(Result.morePR)
      ######################################################
      # end loop over target probabilities

      with open(csvPrefix+"PRmore.csv",'w') as outFile:
         for point in Result.morePR:
            outFile.write(str(point[1])+','+str(point[2]))

      addstatus = str(len(Result.morePR)) + ' PR points written to file: '+ csvPrefix + 'PRmore.csv'
      if verbosity > 0: print(addstatus)
      Result.status = Result.status + '\n' + addstatus
      addstatus = 'lapsed time = ' + putcommas(time.time() - STARTTIME)
      if verbosity > 0: print(addstatus)
      Result.status = Result.status + '\n' + addstatus

      return Result
################################
# LagrangeMorePR ends here
################################

#### start run ####

   AllInOne = False
#   VERYSTARTTIME=time.time()
#   print "##############VERYSTARTTIME:",str(VERYSTARTTIME-VERYSTARTTIME)

##########################
# options defined here
##########################
   try:
      conf_options_parser = construct_ph_options_parser("lagrange [options]")
      conf_options_parser.add_argument("--beta-min",
                                     help="The min beta level for the chance constraint. Default is None",
                                     action="store",
                                     dest="beta_min",
                                     type=float,
                                     default=None)
      conf_options_parser.add_argument("--beta-max",
                                     help="The beta level for the chance constraint. Default is None",
                                     action="store",
                                     dest="beta_max",
                                     type=float,
                                     default=None)
      conf_options_parser.add_argument("--min-prob",
                                     help="Tolerance for testing probability > 0. Default is 1e-5",
                                     action="store",
                                     dest="min_prob",
                                     type=float,
                                     default=1e-5)
      conf_options_parser.add_argument("--beta-tol",
                                     help="Tolerance for testing equality to beta. Default is 10^-2",
                                     action="store",
                                     dest="beta_tol",
                                     type=float,
                                     default=1e-2)
      conf_options_parser.add_argument("--Lagrange-gap",
                                     help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4.",
                                     action="store",
                                     type=float,
                                     dest="Lagrange_gap",
                                     default=0.0001)
      conf_options_parser.add_argument("--max-number",
                                     help="The max number of PR points. Default = 10.",
                                     action="store",
                                     dest="max_number",
                                     type=int,
                                     default=10)
      conf_options_parser.add_argument("--max-time",
                                     help="Maximum time (seconds). Default is 3600.",
                                     action="store",
                                     dest="max_time",
                                     type=float,
                                     default=3600)
      conf_options_parser.add_argument("--csvPrefix",
                                     help="Input file name prefix.  Default is ''",
                                     action="store",
                                     dest="csvPrefix",
                                     type=str,
                                     default="")
      conf_options_parser.add_argument("--lambda-parm-name",
                                     help="The name of the lambda parameter in the model. Default is lambdaMult",
                                     action="store",
                                     dest="lambda_parm_name",
                                     type=str,
                                     default="lambdaMult")
      conf_options_parser.add_argument("--indicator-var-name",
                                     help="The name of the indicator variable for the chance constraint. The default is delta",
                                     action="store",
                                     dest="indicator_var_name",
                                     type=str,
                                     default="delta")
      conf_options_parser.add_argument("--stage-num",
                                     help="The stage number of the CC indicator variable (number, not name). Default is 2",
                                     action="store",
                                     dest="stage_num",
                                     type=int,
                                     default=2)
      conf_options_parser.add_argument("--verbosity",
                                     help="verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.",
                                     action="store",
                                     dest="verbosity",
                                     type=int,
                                     default=1)
      conf_options_parser.add_argument("--prob-file",
                                     help="file name specifiying probabilities",
                                     action="store",
                                     dest="probFileName",
                                     type=str,
                                     default=None)
# The following needed for solve_ph_code in lagrangeutils
      conf_options_parser.add_argument("--solve-with-ph",
                                     help="Perform solves via PH rather than an EF solve. Default is False",
                                     action="store_true",
                                     dest="solve_with_ph",
                                     default=False)

################################################################

      options = conf_options_parser.parse_args(args=args)
      # temporary hack
      options._ef_options = conf_options_parser._ef_options
      options._ef_options.import_argparse(options)
   except SystemExit as _exc:
      # the parser throws a system exit if "-h" is specified - catch
      # it to exit gracefully.
      return _exc.code

   if options.verbose is True:
      print("Loading reference model and scenario tree")

   scenario_instance_factory = \
        ScenarioTreeInstanceFactory(options.model_directory,
                                    options.instance_directory)

   full_scenario_tree = \
            GenerateScenarioTreeForPH(options,
                                      scenario_instance_factory)

   solver_manager = SolverManagerFactory(options.solver_manager_type)
   if solver_manager is None:
      raise ValueError("Failed to create solver manager of "
                       "type="+options.solver_manager_type+
                       " specified in call to PH constructor")
   if isinstance(solver_manager,
                 pyomo.solvers.plugins.smanager.phpyro.SolverManager_PHPyro):
      solver_manager.deactivate()
      raise ValueError("PHPyro can not be used as the solver manager")

   try:

      if (scenario_instance_factory is None) or (full_scenario_tree is None):
         raise RuntimeError("***ERROR: Failed to initialize the model and/or scenario tree data.")

      # load_model gets called again, so lets make sure unarchived directories are used
      options.model_directory = scenario_instance_factory._model_filename
      options.instance_directory = scenario_instance_factory._scenario_tree_filename

      scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)

      # create ph objects for finding the solution. we do this even if
      # we're solving the extensive form

      if options.verbose is True:
         print("Loading scenario instances and initializing scenario tree for full problem.")

########## Here is where multiplier search is called ############
      Result = LagrangeMorePR()
#####################################################################################
   finally:

      solver_manager.deactivate()
      # delete temporary unarchived directories
      scenario_instance_factory.close()

   print("\n====================  returned from LagrangeMorePR")
   print(str(Result.status))
   try:
     print("Envelope:")
     print(str(PrintPRpoints(Result.PRoptimal)))
     print("\nAdded:")
     PrintPRpoints(Result.morePR)
   except:
     print("from run:  PrintPRpoints failed")
     sys.exit()

# combine tables and sort by probability
   if len(Result.morePR) > 0:
     PRpoints = copy.deepcopy(Result.PRoptimal)
     for lbz in Result.morePR: PRpoints.append(lbz)
     print("Combined table of PR points (sorted):")
     PRpoints.sort(key=operator.itemgetter(1))
     print(str(PrintPRpoints(PRpoints)))
Exemple #7
0
class ScenarioTreeServerPyro(TaskWorker):

    # Maps name to a registered worker class to instantiate
    _registered_workers = {}

    @classmethod
    def get_registered_worker_type(cls, name):
        if name in cls._registered_workers:
            return cls._registered_workers[name]
        raise KeyError("No worker type has been registered under the name "
                       "'%s' for ScenarioTreeServerPyro" % (name))

    def __init__(self, *args, **kwds):

        mpi = kwds.pop('mpi', None)
        # add for purposes of diagnostic output.
        kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
                        % (os.getpid(), socket.gethostname()))
        if mpi is not None:
            assert len(mpi) == 2
            kwds["name"] += "_MPIRank_"+str(mpi[1].rank)
        kwds["caller_name"] = kwds["name"]
        self._modules_imported = kwds.pop('modules_imported', {})

        TaskWorker.__init__(self, **kwds)
        assert hasattr(self, "_bulk_task_collection")
        self._bulk_task_collection = True
        self._contiguous_task_processing = False

        self.type = self.WORKERNAME
        self.block = True
        self.timeout = None
        self._worker_map = {}
        self._init_verbose = self._verbose

        # A reference to the mpi4py.MPI namespace
        self.MPI = None
        # The communicator and group associated with all processors
        self.mpi_comm_world = None
        self.mpi_group_world = None
        # The communicator associated with the workers assigned
        # to the current current client
        self.mpi_comm_workers = None
        if mpi is not None:
            assert len(mpi) == 2
            self.MPI = mpi[0]
            self.mpi_comm_world = mpi[1]
            self.mpi_group_world = self.mpi_comm_world.Get_group()

        #
        # These will be used by all subsequent workers created
        # by this server. Their creation can eat up a nontrivial
        # amount of initialization time when a large number of
        # workers are created on this server, so we only create
        # them once.
        #
        self._scenario_instance_factory = None
        self._full_scenario_tree = None

    def reset(self):
        if self._scenario_instance_factory is not None:
            self._scenario_instance_factory.close()
        self._scenario_instance_factory = None
        self._full_scenario_tree = None
        for worker_name in list(self._worker_map):
            self.remove_worker(worker_name)
        if self.mpi_comm_workers is not None:
            self.mpi_comm_workers.Free()
            self.mpi_comm_workers = None
        self._verbose = self._init_verbose

    def remove_worker(self, name):
        self._worker_map[name].close()
        del self._worker_map[name]

    def process(self, data):
        self._worker_task_return_queue = self._current_task_client
        try:
            # The only reason we are go through this much
            # effort to deal with the serpent serializer
            # is because it is the default in Pyro4.
            if using_pyro4 and \
               (Pyro4.config.SERIALIZER == 'serpent'):
                if six.PY3:
                    assert type(data) is dict
                    assert data['encoding'] == 'base64'
                    data = base64.b64decode(data['data'])
                else:
                    assert type(data) is unicode
                    data = str(data)
            return pickle.dumps(self._process(pickle.loads(data)))
        except:
            logger.error(
                "Scenario tree server %s caught an exception of type "
                "%s while processing a task. Going idle."
                % (self.WORKERNAME, sys.exc_info()[0].__name__))
            traceback.print_exception(*sys.exc_info())
            self._worker_error = True
            return pickle.dumps(TaskProcessingError(traceback.format_exc()))

    def _process(self, data):
        data = pyutilib.misc.Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            #with PauseGC() as pgc:
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            model_input = data.options.pop('model', None)
            if model_input is None:
                model_input = data.options.pop('model_callback')
                assert dill_available
                model_input = dill.loads(model_input)

            scenario_tree_input = data.options.pop('scenario_tree')
            data_input = data.options.pop('data')
            mpi_group = data.options.pop("mpi_group",None)
            verbose = data.options.pop("verbose", False)
            assert len(data.options) == 0
            self._verbose |= verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._verbose:
                print("Server %s received setup request."
                      % (self.WORKERNAME))

            # Make sure these are not archives
            assert (not isinstance(model_input, six.string_types)) or \
                os.path.exists(model_input)
            assert isinstance(scenario_tree_input, ScenarioTree)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    model_input,
                    scenario_tree_input,
                    data=data_input)

            #
            # Try to prevent unnecessarily re-importing the model module
            # if other callbacks are in the same location. Doing so might
            # have serious consequences.
            #
            if self._scenario_instance_factory._model_module is not None:
                self._modules_imported[self._scenario_instance_factory.\
                                       _model_filename] = \
                    self._scenario_instance_factory._model_module
            assert self._scenario_instance_factory._scenario_tree_module is None

            self._full_scenario_tree = \
                 self._scenario_instance_factory.generate_scenario_tree()

            assert self.mpi_comm_workers is None
            if self.mpi_comm_world is not None:
                assert self.mpi_group_world is not None
                assert mpi_group is not None
                mpi_group = self.mpi_group_world.Incl(mpi_group)
                self.mpi_comm_workers = \
                    self.mpi_comm_world.Create_group(mpi_group)
            else:
                assert mpi_group is None

            if self._full_scenario_tree is None:
                 raise RuntimeError("Unable to launch scenario tree worker - "
                                    "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s."
                      % (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name."
                     % (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]

            self._worker_map[worker_name] = worker_type(
                self,
                worker_name,
                *data.init_args,
                **data.init_kwds)
            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._verbose:
                print("Server %s releasing worker: %s"
                      % (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._verbose:
                print("Server %s received reset request"
                      % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._verbose:
                print("Server %s received shutdown request"
                      % (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s"
                             % (self.WORKERNAME, data.action))

        return result
Exemple #8
0
    def _process(self, data):
        data = pyutilib.misc.Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            #with PauseGC() as pgc:
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            model_input = data.options.pop('model', None)
            if model_input is None:
                model_input = data.options.pop('model_callback')
                assert dill_available
                model_input = dill.loads(model_input)

            scenario_tree_input = data.options.pop('scenario_tree')
            data_input = data.options.pop('data')
            mpi_group = data.options.pop("mpi_group",None)
            verbose = data.options.pop("verbose", False)
            assert len(data.options) == 0
            self._verbose |= verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._verbose:
                print("Server %s received setup request."
                      % (self.WORKERNAME))

            # Make sure these are not archives
            assert (not isinstance(model_input, six.string_types)) or \
                os.path.exists(model_input)
            assert isinstance(scenario_tree_input, ScenarioTree)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    model_input,
                    scenario_tree_input,
                    data=data_input)

            #
            # Try to prevent unnecessarily re-importing the model module
            # if other callbacks are in the same location. Doing so might
            # have serious consequences.
            #
            if self._scenario_instance_factory._model_module is not None:
                self._modules_imported[self._scenario_instance_factory.\
                                       _model_filename] = \
                    self._scenario_instance_factory._model_module
            assert self._scenario_instance_factory._scenario_tree_module is None

            self._full_scenario_tree = \
                 self._scenario_instance_factory.generate_scenario_tree()

            assert self.mpi_comm_workers is None
            if self.mpi_comm_world is not None:
                assert self.mpi_group_world is not None
                assert mpi_group is not None
                mpi_group = self.mpi_group_world.Incl(mpi_group)
                self.mpi_comm_workers = \
                    self.mpi_comm_world.Create_group(mpi_group)
            else:
                assert mpi_group is None

            if self._full_scenario_tree is None:
                 raise RuntimeError("Unable to launch scenario tree worker - "
                                    "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s."
                      % (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name."
                     % (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]

            self._worker_map[worker_name] = worker_type(
                self,
                worker_name,
                *data.init_args,
                **data.init_kwds)
            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._verbose:
                print("Server %s releasing worker: %s"
                      % (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._verbose:
                print("Server %s received reset request"
                      % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._verbose:
                print("Server %s received shutdown request"
                      % (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s"
                             % (self.WORKERNAME, data.action))

        return result
Exemple #9
0
    def __init__(self, fsfile,
                 fsfct = None,
                 tree_model = None,
                 phopts = None):
        """Initialize a StochSolver object.
        """
        if fsfct is None:
            # Changed in October 2018: None implies AbstractModel
            args_list = _optiondict_2_list(phopts)
            parser = phinit.construct_ph_options_parser("")
            options = parser.parse_args(args_list)

            scenario_instance_factory = \
                ScenarioTreeInstanceFactory(fsfile, tree_model)

            try:
                self.scenario_tree = \
                    phinit.GenerateScenarioTreeForPH(options,
                                                     scenario_instance_factory)
            except:
                print ("ERROR in StochSolver called from",inspect.stack()[1][3])
                raise RuntimeError("fsfct is None, so assuming",
                      "AbstractModel but could not find all ingredients.")
                
        else:  # concrete model
            if  callable(fsfct):
                scen_function = fsfct
            else: # better be a string
                fsfile = fsfile.replace('.py','')  # import does not like .py
                # __import__ only gives the top level module
                # probably need to be dealing with modules installed via setup.py
                m = __import__(fsfile)
                for n in fsfile.split(".")[1:]:
                    m = getattr(m, n)
                scen_function = getattr(m, fsfct)

            if tree_model is None:
                treecbname = "pysp_scenario_tree_model_callback"
                tree_maker = getattr(m, treecbname)

                tree = tree_maker()
                if isinstance(tree, Pyo.ConcreteModel):
                    tree_model = tree
                else:
                    raise RuntimeError("The tree returned by",treecbname,
                                       "must be a ConcreteModel") 
                    
                scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model)

            else: 
                # DLW March 21: still not correct
                scenario_instance_factory = \
                    ScenarioTreeInstanceFactory(scen_function, tree_model)


            kwargs = _kwfromphopts(phopts)
            self.scenario_tree = \
                scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
            instances = scenario_instance_factory. \
                        construct_instances_for_scenario_tree(self.scenario_tree)
            self.scenario_tree.linkInInstances(instances)        
Exemple #10
0
def run(args=None):
##########################================================#########
   # to import plugins
   import pyomo.environ
   import pyomo.solvers.plugins.smanager.phpyro
   import pyomo.solvers.plugins.smanager.pyro

   def partialLagrangeParametric(args=None):
      print("lagrangeParam begins ")
      blanks = "                          "  # used for formatting print statements
      class Object(object): pass
      Result = Object()

# options used
      IndVarName = options.indicator_var_name
      CCStageNum = options.stage_num
      alphaTol = options.alpha_tol
      MaxMorePR = options.MaxMorePR # option to include up to this many PR points above F^* with all delta fixed
      outputFilePrefix = options.outputFilePrefix

# We write ScenarioList = name, probability
#          PRoptimal    = probability, min-cost, [selections]
#          PRmore       = probability, min-cost, [selections]
# ================ sorted by probability ========================
#
# These can be read to avoid re-computing points

      ph = PHFromScratch(options)
      Result.ph = ph
      rootnode = ph._scenario_tree._stages[0]._tree_nodes[0]   # use rootnode to loop over scenarios

      if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing():
         print("We are solving a MINIMIZATION problem.\n")
      else:
         print("We are solving a MAXIMIZATION problem.\n")

# initialize
      ScenarioList = []
      lambdaval = 0.
      lagrUtil.Set_ParmValue(ph,
                             options.lambda_parm_name,
                             lambdaval)

      # IMPORTANT: Preprocess the scenario instances
      #            before fixing variables, otherwise they
      #            will be preprocessed out of the expressions
      #            and the output_fixed_variable_bounds option
      #            will have no effect when we update the
      #            fixed variable values (and then assume we
      #            do not need to preprocess again because
      #            of this option).
      ph._preprocess_scenario_instances()

      lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0)
      for scenario in rootnode._scenarios:
         ScenarioList.append((scenario._name,
                              scenario._probability))

      # sorts from min to max probability
      ScenarioList.sort(key=operator.itemgetter(1))
      with open(outputFilePrefix+'ScenarioList.csv','w') as outFile:
         for scenario in ScenarioList:
            outFile.write(scenario[0]+ ", " +str(scenario[1])+"\n")
      Result.ScenarioList = ScenarioList

      print("lambda= "+str(lambdaval)+" ...run begins "+str(len(ScenarioList))+" scenarios")
      SolStat, zL = lagrUtil.solve_ph_code(ph, options)
      print("\t...ends")
      bL = Compute_ExpectationforVariable(ph,
                                          IndVarName,
                                          CCStageNum)
      if bL > 0:
         print("** bL = "+str(bL)+"  > 0")
         return Result

      print("Initial cost = "+str(zL)+"  for bL = "+str(bL))

      lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1)

      print("lambda= "+str(lambdaval)+" ...run begins")
      SolStat, zU = lagrUtil.solve_ph_code(ph, options)
      print("\t...ends")
      bU = Compute_ExpectationforVariable(ph,
                                          IndVarName,
                                          CCStageNum)
      if bU < 1:
            print("** bU = "+str(bU)+"  < 1")

      lagrUtil.FreeAllIndicatorVariables(ph, IndVarName)

      Result.lbz = [ [0,bL,zL], [None,bU,zU] ]
      Result.selections = [[], ScenarioList]
      NumIntervals = 1
      print("initial gap = "+str(1-zL/zU)+" \n")
      print("End of test; this is only a test.")

      return Result
################################
# LagrangeParametric ends here
################################

#### start run ####

   AllInOne = False

##########################
# options defined here
##########################
   try:
      conf_options_parser = construct_ph_options_parser("lagrange [options]")
      conf_options_parser.add_argument("--alpha",
                                     help="The alpha level for the chance constraint. Default is 0.05",
                                     action="store",
                                     dest="alpha",
                                     type=float,
                                     default=0.05)
      conf_options_parser.add_argument("--alpha-min",
                                     help="The min alpha level for the chance constraint. Default is None",
                                     action="store",
                                     dest="alpha_min",
                                     type=float,
                                     default=None)
      conf_options_parser.add_argument("--alpha-max",
                                     help="The alpha level for the chance constraint. Default is None",
                                     action="store",
                                     dest="alpha_max",
                                     type=float,
                                     default=None)
      conf_options_parser.add_argument("--min-prob",
                                     help="Tolerance for testing probability > 0. Default is 1e-5",
                                     action="store",
                                     dest="min_prob",
                                     type=float,
                                     default=1e-5)
      conf_options_parser.add_argument("--alpha-tol",
                                     help="Tolerance for testing equality to alpha. Default is 1e-5",
                                     action="store",
                                     dest="alpha_tol",
                                     type=float,
                                     default=1e-5)
      conf_options_parser.add_argument("--MaxMorePR",
                                     help="Generate up to this many additional PR points after response function. Default is 0",
                                     action="store",
                                     dest="MaxMorePR",
                                     type=int,
                                     default=0)
      conf_options_parser.add_argument("--outputFilePrefix",
                                     help="Output file name.  Default is ''",
                                     action="store",
                                     dest="outputFilePrefix",
                                     type=str,
                                     default="")
      conf_options_parser.add_argument("--stage-num",
                                     help="The stage number of the CC indicator variable (number, not name). Default is 2",
                                     action="store",
                                     dest="stage_num",
                                     type=int,
                                     default=2)
      conf_options_parser.add_argument("--lambda-parm-name",
                                     help="The name of the lambda parameter in the model. Default is lambdaMult",
                                     action="store",
                                     dest="lambda_parm_name",
                                     type=str,
                                     default="lambdaMult")
      conf_options_parser.add_argument("--indicator-var-name",
                                     help="The name of the indicator variable for the chance constraint. The default is delta",
                                     action="store",
                                     dest="indicator_var_name",
                                     type=str,
                                     default="delta")
      conf_options_parser.add_argument("--use-Loane-cuts",
                                     help="Add the Loane cuts if there is a gap. Default is False",
                                     action="store_true",
                                     dest="add_Loane_cuts",
                                     default=False)
      conf_options_parser.add_argument("--fofx-var-name",
                                     help="(Loane) The name of the model's auxiliary variable that is constrained to be f(x). Default is fofox",
                                     action="store",
                                     dest="fofx_var_name",
                                     type=str,
                                     default="fofx")
      conf_options_parser.add_argument("--solve-with-ph",
                                     help="Perform solves via PH rather than an EF solve. Default is False",
                                     action="store_true",
                                     dest="solve_with_ph",
                                     default=False)
      conf_options_parser.add_argument("--skip-graph",
                                     help="Do not show the graph at the end. Default is False (i.e. show the graph)",
                                     action="store_true",
                                     dest="skip_graph",
                                     default=False)
      conf_options_parser.add_argument("--write-xls",
                                     help="Write results into a xls file. Default is False",
                                     action="store_true",
                                     dest="write_xls",
                                     default=False)
      conf_options_parser.add_argument("--skip-ExpFlip",
                                     help="Do not show the results for flipping the indicator variable for each scenario. Default is False (i.e. show the flipping-results)",
                                     action="store_true",
                                     dest="skip_ExpFlip",
                                     default=False)
      conf_options_parser.add_argument("--HeurFlip",
                                     help="The number of solutions to evaluate after the heuristic. Default is 3. For 0 the heuristic flip gets skipped.",
                                     action="store",
                                     type=int,
                                     dest="HeurFlip",
                                     default=3)
      conf_options_parser.add_argument("--HeurMIP",
                                     help="The mipgap for the scenariowise solves in the heuristic. Default is 0.0001",
                                     action="store",
                                     type=float,
                                     dest="HeurMIP",
                                     default=0.0001)
      conf_options_parser.add_argument("--interactive",
                                     help="Enable interactive version of the code. Default is False.",
                                     action="store_true",
                                     dest="interactive",
                                     default=False)
      conf_options_parser.add_argument("--Lgap",
                                     help="The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4",
                                     action="store",
                                     type=float,
                                     dest="LagrangeGap",
                                     default=0.0001)
      conf_options_parser.add_argument("--lagrange-method",
                                     help="The Lagrange multiplier search method",
                                     action="store",
                                     dest="lagrange_search_method",
                                     type=str,
                                     default="tangential")
      conf_options_parser.add_argument("--max-lambda",
                                     help="The max value of the multiplier. Default=10^10",
                                     action="store",
                                     dest="max_lambda",
                                     type=float,
                                     default=10**10)
      conf_options_parser.add_argument("--min-lambda",
                                     help="The min value of the multiplier. Default=0.0",
                                     action="store",
                                     dest="min_lambda",
                                     type=float,
                                     default=0)
      conf_options_parser.add_argument("--min-probability",
                                     help="The min value of scenario probability. Default=10^-15",
                                     action="store",
                                     dest="min_probability",
                                     type=float,
                                     default=10**(-15))

################################################################

      options = conf_options_parser.parse_args(args=args)
      # temporary hack
      options._ef_options = conf_options_parser._ef_options
      options._ef_options.import_argparse(options)
   except SystemExit as _exc:
      # the parser throws a system exit if "-h" is specified - catch
      # it to exit gracefully.
      return _exc.code

   # load the reference model and create the scenario tree - no
   # scenario instances yet.
   if options.verbose:
      print("Loading reference model and scenario tree")
   #scenario_instance_factory, full_scenario_tree = load_models(options)
   scenario_instance_factory = \
        ScenarioTreeInstanceFactory(options.model_directory,
                                    options.instance_directory)

   full_scenario_tree = \
            GenerateScenarioTreeForPH(options,
                                      scenario_instance_factory)

   solver_manager = SolverManagerFactory(options.solver_manager_type)
   if solver_manager is None:
      raise ValueError("Failed to create solver manager of "
                       "type="+options.solver_manager_type+
                       " specified in call to PH constructor")
   if isinstance(solver_manager,
                 pyomo.solvers.plugins.smanager.phpyro.SolverManager_PHPyro):
      raise ValueError("PHPyro can not be used as the solver manager")

   try:

      if (scenario_instance_factory is None) or (full_scenario_tree is None):
         raise RuntimeError("***ERROR: Failed to initialize model and/or the scenario tree data.")

      # load_model gets called again, so lets make sure unarchived directories are used
      options.model_directory = scenario_instance_factory._model_filename
      options.instance_directory = scenario_instance_factory._scenario_tree_filename

      scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes)

      # create ph objects for finding the solution. we do this even if
      # we're solving the extensive form

      if options.verbose:
         print("Loading scenario instances and initializing scenario tree for full problem.")

########## Here is where multiplier search is called ############
      Result = partialLagrangeParametric()
#####################################################################################

   finally:
      # delete temporary unarchived directories
      scenario_instance_factory.close()

   print("\nreturned from partialLagrangeParametric")
Exemple #11
0
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker):

    # Maps name to a registered worker class to instantiate
    _registered_workers = {}

    @classmethod
    def get_registered_worker_type(cls, name):
        if name in cls._registered_workers:
            return cls._registered_workers[name]
        raise KeyError("No worker type has been registered under the name "
                       "'%s' for ScenarioTreeServerPyro" % (name))

    def __init__(self, *args, **kwds):

        mpi = kwds.pop('mpi', None)
        # add for purposes of diagnostic output.
        kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" %
                        (os.getpid(), socket.gethostname()))
        if mpi is not None:
            assert len(mpi) == 2
            kwds["name"] += "_MPIRank_" + str(mpi[1].rank)
        kwds["caller_name"] = kwds["name"]
        self._modules_imported = kwds.pop('modules_imported', {})

        pyu_pyro.TaskWorker.__init__(self, **kwds)
        assert hasattr(self, "_bulk_task_collection")
        self._bulk_task_collection = True
        self._contiguous_task_processing = False

        self.type = self.WORKERNAME
        self.block = True
        self.timeout = None
        self._worker_map = {}
        self._init_verbose = self._verbose

        # A reference to the mpi4py.MPI namespace
        self.MPI = None
        # The communicator and group associated with all processors
        self.mpi_comm_world = None
        self.mpi_group_world = None
        # The communicator associated with the workers assigned
        # to the current current client
        self.mpi_comm_workers = None
        if mpi is not None:
            assert len(mpi) == 2
            self.MPI = mpi[0]
            self.mpi_comm_world = mpi[1]
            self.mpi_group_world = self.mpi_comm_world.Get_group()

        #
        # These will be used by all subsequent workers created
        # by this server. Their creation can eat up a nontrivial
        # amount of initialization time when a large number of
        # workers are created on this server, so we only create
        # them once.
        #
        self._scenario_instance_factory = None
        self._full_scenario_tree = None

    def reset(self):
        if self._scenario_instance_factory is not None:
            self._scenario_instance_factory.close()
        self._scenario_instance_factory = None
        self._full_scenario_tree = None
        for worker_name in list(self._worker_map):
            self.remove_worker(worker_name)
        if self.mpi_comm_workers is not None:
            self.mpi_comm_workers.Free()
            self.mpi_comm_workers = None
        self._verbose = self._init_verbose

    def remove_worker(self, name):
        self._worker_map[name].close()
        del self._worker_map[name]

    def process(self, data):
        self._worker_task_return_queue = self._current_task_client
        try:
            # The only reason we are go through this much
            # effort to deal with the serpent serializer
            # is because it is the default in Pyro4.
            if pyu_pyro.using_pyro4 and \
               (Pyro4.config.SERIALIZER == 'serpent'):
                if six.PY3:
                    assert type(data) is dict
                    assert data['encoding'] == 'base64'
                    data = base64.b64decode(data['data'])
                else:
                    assert type(data) is unicode
                    data = str(data)
            return pickle.dumps(self._process(pickle.loads(data)))
        except:
            logger.error("Scenario tree server %s caught an exception of type "
                         "%s while processing a task. Going idle." %
                         (self.WORKERNAME, sys.exc_info()[0].__name__))
            traceback.print_exception(*sys.exc_info())
            self._worker_error = True
            return pickle.dumps(
                pyu_pyro.TaskProcessingError(traceback.format_exc()))

    def _process(self, data):
        data = Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            model_input = data.options.pop('model', None)
            if model_input is None:
                model_input = data.options.pop('model_callback')
                assert dill_available
                model_input = dill.loads(model_input)

            scenario_tree_input = data.options.pop('scenario_tree')
            data_input = data.options.pop('data')
            mpi_group = data.options.pop("mpi_group", None)
            verbose = data.options.pop("verbose", False)
            assert len(data.options) == 0
            self._verbose |= verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._verbose:
                print("Server %s received setup request." % (self.WORKERNAME))

            # Make sure these are not archives
            assert (not isinstance(model_input, six.string_types)) or \
                os.path.exists(model_input)
            assert isinstance(scenario_tree_input, ScenarioTree)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    model_input,
                    scenario_tree_input,
                    data=data_input)

            #
            # Try to prevent unnecessarily re-importing the model module
            # if other callbacks are in the same location. Doing so might
            # have serious consequences.
            #
            if self._scenario_instance_factory._model_module is not None:
                self._modules_imported[self._scenario_instance_factory.\
                                       _model_filename] = \
                    self._scenario_instance_factory._model_module
            assert self._scenario_instance_factory._scenario_tree_module is None

            self._full_scenario_tree = \
                 self._scenario_instance_factory.generate_scenario_tree()

            assert self.mpi_comm_workers is None
            if self.mpi_comm_world is not None:
                assert self.mpi_group_world is not None
                assert mpi_group is not None
                mpi_group = self.mpi_group_world.Incl(mpi_group)
                self.mpi_comm_workers = \
                    self.mpi_comm_world.Create_group(mpi_group)
            else:
                assert mpi_group is None

            if self._full_scenario_tree is None:
                raise RuntimeError("Unable to launch scenario tree worker - "
                                   "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s." %
                      (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name." %
                    (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]

            self._worker_map[worker_name] = worker_type(
                self, worker_name, *data.init_args, **data.init_kwds)
            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._verbose:
                print("Server %s releasing worker: %s" %
                      (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._verbose:
                print("Server %s received reset request" % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._verbose:
                print("Server %s received shutdown request" %
                      (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s" %
                             (self.WORKERNAME, data.action))

        return result
Exemple #12
0
    def _process(self, data):
        data = Bunch(**data)
        result = None
        if not data.action.startswith('ScenarioTreeServerPyro_'):
            result = getattr(self._worker_map[data.worker_name], data.action)\
                     (*data.args, **data.kwds)

        elif data.action == 'ScenarioTreeServerPyro_setup':
            model_input = data.options.pop('model', None)
            if model_input is None:
                model_input = data.options.pop('model_callback')
                assert dill_available
                model_input = dill.loads(model_input)

            scenario_tree_input = data.options.pop('scenario_tree')
            data_input = data.options.pop('data')
            mpi_group = data.options.pop("mpi_group", None)
            verbose = data.options.pop("verbose", False)
            assert len(data.options) == 0
            self._verbose |= verbose
            assert self._scenario_instance_factory is None
            assert self._full_scenario_tree is None
            if self._verbose:
                print("Server %s received setup request." % (self.WORKERNAME))

            # Make sure these are not archives
            assert (not isinstance(model_input, six.string_types)) or \
                os.path.exists(model_input)
            assert isinstance(scenario_tree_input, ScenarioTree)
            self._scenario_instance_factory = \
                ScenarioTreeInstanceFactory(
                    model_input,
                    scenario_tree_input,
                    data=data_input)

            #
            # Try to prevent unnecessarily re-importing the model module
            # if other callbacks are in the same location. Doing so might
            # have serious consequences.
            #
            if self._scenario_instance_factory._model_module is not None:
                self._modules_imported[self._scenario_instance_factory.\
                                       _model_filename] = \
                    self._scenario_instance_factory._model_module
            assert self._scenario_instance_factory._scenario_tree_module is None

            self._full_scenario_tree = \
                 self._scenario_instance_factory.generate_scenario_tree()

            assert self.mpi_comm_workers is None
            if self.mpi_comm_world is not None:
                assert self.mpi_group_world is not None
                assert mpi_group is not None
                mpi_group = self.mpi_group_world.Incl(mpi_group)
                self.mpi_comm_workers = \
                    self.mpi_comm_world.Create_group(mpi_group)
            else:
                assert mpi_group is None

            if self._full_scenario_tree is None:
                raise RuntimeError("Unable to launch scenario tree worker - "
                                   "scenario tree construction failed.")

            result = True

        elif data.action == "ScenarioTreeServerPyro_initialize":

            worker_name = data.worker_name
            if self._verbose:
                print("Server %s received request to initialize "
                      "scenario tree worker with name %s." %
                      (self.WORKERNAME, worker_name))

            assert self._scenario_instance_factory is not None
            assert self._full_scenario_tree is not None

            if worker_name in self._worker_map:
                raise RuntimeError(
                    "Server %s Cannot initialize worker with name '%s' "
                    "because a worker already exists with that name." %
                    (self.WORKERNAME, worker_name))

            worker_type = self._registered_workers[data.worker_type]

            self._worker_map[worker_name] = worker_type(
                self, worker_name, *data.init_args, **data.init_kwds)
            result = True

        elif data.action == "ScenarioTreeServerPyro_release":

            if self._verbose:
                print("Server %s releasing worker: %s" %
                      (self.WORKERNAME, data.worker_name))
            self.remove_worker(data.worker_name)
            result = True

        elif data.action == "ScenarioTreeServerPyro_reset":

            if self._verbose:
                print("Server %s received reset request" % (self.WORKERNAME))
            self.reset()
            result = True

        elif data.action == "ScenarioTreeServerPyro_shutdown":

            if self._verbose:
                print("Server %s received shutdown request" %
                      (self.WORKERNAME))
            self.reset()
            self._worker_shutdown = True
            result = True

        else:
            raise ValueError("Server %s: Invalid command: %s" %
                             (self.WORKERNAME, data.action))

        return result