def LagrangeParametric(args=None): class Object(object): pass Result = Object() Result.status = 'LagrangeParam begins '+ datetime_string() + '...running new ph' ph = None blanks = " " # used for formatting print statements # options used betaMin = options.beta_min betaMax = options.beta_max betaTol = options.beta_tol gapTol = options.Lagrange_gap minProb = options.min_prob maxIntervals = options.max_intervals maxTime = options.max_time IndVarName = options.indicator_var_name multName = options.lambda_parm_name CCStageNum = options.stage_num csvPrefix = options.csvPrefix verbosity = options.verbosity verbosity = 2 # override for debug (= 3 to get super-debug) HGdebug = 0 # special debug (not public) # local...may become option optTol = gapTol #################################################################### STARTTIME = time.time() Result.status = "options set" if verbosity > 1: print("From LagrangeParametric, status = %s\tSTARTTIME = %s" \ % (str(getattr(Result,'status')), str(STARTTIME))) ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios ReferenceInstance = ph._instances[rootnode._scenarios[0]._name] # arbitrary scenario if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing(): sense = 'min' else: sense = 'max' scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) if options.verbosity > 0: print("%s %s scenarios" % (str(sense),str(scenario_count))) # initialize Result.status = 'starting at '+datetime_string() if verbosity > 0: print(Result.status) ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, multName,lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() sumprob = 0. minprob = 1. maxprob = 0. # fixed = 0 to get PR point at b=0 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: instance = ph._instances[scenario._name] sname = scenario._name sprob = scenario._probability sumprob = sumprob + sprob minprob = min(minprob,sprob) maxprob = max(maxprob,sprob) ScenarioList.append([sname,sprob]) ScenarioList.sort(key=operator.itemgetter(1)) # sorts from min to max probability if verbosity > 0: print("probabilities sum to %f range: %f to %f" % (sumprob,minprob,maxprob)) Result.ScenarioList = ScenarioList # Write ScenarioList = name, probability in csv file sorted by probability outName = csvPrefix + 'ScenarioList.csv' print("writing to %s" % outName) with open(outName,'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0]+", "+str(scenario[1])+'\n') Result.ScenarioList = ScenarioList addstatus = 'Scenario List written to ' + csvPrefix+'ScenarioList.csv' Result.status = Result.status + '\n' + addstatus if verbosity > 0: print(addstatus) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %f" % lambdaval) SolStat, zL = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zL = %s" % str(zL)) bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = %s > 0 (all %s = 0)" % (str(bL), str(IndVarName))) return Result if verbosity > 0: print("Initial optimal obj = %s for bL = %s" % (str(zL), str(bL))) # fixed = 1 to get PR point at b=1 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %s" % str(lambdaval)) SolStat, zU = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zU = %s" % str(zU)) if not SolStat[0:2] == 'ok': print(str(SolStat[0:3])+" is not 'ok'") addstatus = "** Solution is non-optimal...aborting" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1.- betaTol and verbosity > 0: print("** Warning: bU = %s < 1" % str(bU)) ### enumerate points in PR space (all but one scenario) # Result.lbz = [ [0,bL,zL], [None,bU,zU] ] # for scenario in rootnode._scenarios: # sname = scenario._name # instance = ph._instances[sname] # print "excluding scenario",sname # getattr(instance,IndVarName).value = 0 # print sname,"value =",getattr(instance,IndVarName).value,getattr(instance,IndVarName).fixed # SolStat, z = lagrUtil.solve_ph_code(ph, options) # b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) # print "solve ends with status =",SolStat,"(b, z) =",b,z # getattr(instance,IndVarName).value = 1 # Result.lbz.append([None,b,z]) # for t in instance.TimePeriods: # print "Global at",t,"=",instance.posGlobalLoadGenerateMismatch[t].value, \ # '-',instance.negGlobalLoadGenerateMismatch[t].value,"=",\ # instance.GlobalLoadGenerateMismatch[t].value,\ # "\tDemand =",instance.TotalDemand[t].value, ",",\ # "Reserve =",instance.ReserveRequirement[t].value # # PrintPRpoints(Result.lbz) # return Result #### end enumeration ######################################################################## if verbosity > 1: print("We have bU = %s ...about to free all %s for %d scenarios" % \ (str(bU), str(IndVarName), len(ScenarioList))) # free scenario selection variable lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) if verbosity > 1: print("\tall %s freed; elapsed time = %f" % (str(IndVarName), time.time() - STARTTIME)) # initialize with the two endpoints Result.lbz = [ [0.,bL,zL], [None,bU,zU] ] Result.selections = [[], ScenarioList] NumIntervals = 1 if verbosity > 0: print("Initial relative Lagrangian gap = %f maxIntervals = %d" % (1-zL/zU, maxIntervals)) if verbosity > 1: print("entering while loop %s" % datetime_string()) print("\n") ############ main loop to search intervals ############# ######################################################## while NumIntervals < maxIntervals: lapsedTime = time.time() - STARTTIME if lapsedTime > maxTime: addstatus = '** max time reached ' + str(lapsedTime) print(addstatus) Result.status = Result.status + '\n' + addstatus break if verbosity > 1: print("Top of while with %d intervals elapsed time = %f" % (NumIntervals, lapsedTime)) PrintPRpoints(Result.lbz) lambdaval = None ### loop over PR points to find first unfathomed interval to search ### for PRpoint in range(1,len(Result.lbz)): if Result.lbz[PRpoint][0] == None: # multiplier = None means interval with upper endpoint at PRpoint not fathomed bL = Result.lbz[PRpoint-1][1] zL = Result.lbz[PRpoint-1][2] bU = Result.lbz[PRpoint][1] zU = Result.lbz[PRpoint][2] lambdaval = (zU - zL) / (bU - bL) break ############################# # Exited from the for loop if verbosity > 1: print("exited for loop with PRpoint = %s ...lambdaval = %s" % (PRpoint, lambdaval)) if lambdaval == None: break # all intervals are fathomed if verbosity > 1: PrintPRpoints(Result.lbz) if verbosity > 0: print("Searching for b in [%s, %s] with %s = %f" % (str(round(bL,4)), str(round(bU,4)), multName, lambdaval)) # search interval (bL,bU) lagrUtil.Set_ParmValue(ph, multName,lambdaval) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- %s = %f" % (multName, lambdaval)) ######################################################### SolStat, Lagrangian = lagrUtil.solve_ph_code(ph, options) ######################################################### if not SolStat[0:2] == 'ok': addstatus = "** Solution status " + SolStat + " is not optimal" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) z = Lagrangian + lambdaval*b if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- Lagrangian = %f" % Lagrangian) print("\t- b = %s" % str(b)) print("\t- z = %s" % str(z)) print("\n") # We have PR point (b,z), which may be new or one of the endpoints ################################################################## ######### Begin tolerance tests ########## # Test that b is in [bL,bU] if verbosity > 1: print("\ttesting b") if b < bL - betaTol or b > bU + betaTol: addstatus = "** fatal error: probability (= " + str(b) + \ ") is outside interval, (" + str(bL) + ", " + str(bU) + ")" addstatus = addstatus + "\n\t(tolerance = " + str(betaTol) + ")" print(addstatus+'\n') Result.status = Result.status + addstatus return Result # Test that z is in [zL,zU] if verbosity > 1: print("\ttesting z") # using optTol as absolute tolerance (not relative) # ...if we reconsider, need to allow negative z-values if z < zL - optTol or z > zU + optTol: addstatus = "** fatal error: obj (= " + str(z) + \ ") is outside interval, (" + str(zL) + ", " + str(zU) + ")" print(addstatus+'\n') Result.status = Result.status + addstatus return Result # Ok, we have (b,z) in [(bL,zL), (bU,zU)], at least within tolerances oldLagrangian = zL - lambdaval*bL # ensure lambdaval set such that endpoints have same Lagrangian value # (this is probably unnecessary, but check anyway) if abs(oldLagrangian - (zU - lambdaval*bU)) > optTol*abs(oldLagrangian): addstatus = "** fatal error: Lagrangian at (bL,zL) = " + \ str(oldLagrangian) + " not= " + str(zU-lambdaval*bU) + \ "\n\t(optTol = " + str(optTol) + ")" Result.status = Result.status + addstatus return Result # no more fatal error tests...need to know if (b,z) is endpoint or new if verbosity > 1: print("No anomalies...testing if b = bL or bU") # Test if endpoint is an alternative optimum of Lagrangian # ...using optTol as *relative* tolerance # (could use other reference values -- eg, avg or max of old and new Lagrangian values) refValue = max( min( abs(oldLagrangian), abs(Lagrangian) ), 1.) alternativeOpt = abs( oldLagrangian - Lagrangian ) <= optTol*refValue # alternativeOpt = True means we computed point (b,z) is alternative optimum such that: # case 1: (b,z) = endpoint, in which case we simply fathom [bL,bU] by setting PRpoint # to [lambdaval,bU,zU] (the numeric value of multiplier means fathomed) # case 2: (b,z) is new PR point on line segment, in which case we split into # [bL,b] and [b,bU], with both fathomed if verbosity > 1: print("oldLagrangian = %s" % str(oldLagrangian)) if alternativeOpt: print(":= Lagrangian = %s" % str(Lagrangian )) else: print("> Lagrangian = %s" % str(Lagrangian)) if alternativeOpt: # setting multiplier of (bU,zU) to a numeric fathoms the interval [bL,bU] Result.lbz[PRpoint][0] = lambdaval # test if (b,z) is an endpoint newPRpoint = abs(b-bL) > betaTol and abs(b-bU) > betaTol if not newPRpoint: # ...(b,z) is NOT an endpoint (or sufficiently close), so split and fathom if verbosity > 1: print("\tnot an endpoint\tlbz = %s" % str(Result.lbz[PRpoint])) if verbosity > 0: print("Lagangian solution is new PR point on line segment of (" \ + str(bL) + ", " + str(bU) +")") print("\tsplitting (bL,bU) into (bL,b) and (b,bU), both fathomed") # note: else ==> b = bL or bU, so we do nothing, having already fathomed [bL,bU] # (b,z) is new PR point, so split interval (still in while loop) ########################################## # alternative optimum ==> split & fathom: (bL,b), (b,bU) if verbosity > 1: print("\talternativeOpt %s newPRpoint = %s" % (alternativeOpt, newPRpoint)) if newPRpoint: NumIntervals += 1 if alternativeOpt: if verbosity > 1: print("\tInsert [lambdaval,b,z] at %f" % PRpoint) Result.lbz = Insert([lambdaval,b,z],PRpoint,Result.lbz) addstatus = "Added PR point on line segment of envelope" if verbosity > 0: print(addstatus+'\n') else: if verbosity > 1: print("\tInsert [None,b,z] at %f" % PRpoint) Result.lbz = Insert([None,b,z],PRpoint,Result.lbz) addstatus = "new envelope extreme point added (interval split, not fathomed)" Result.status = Result.status + "\n" + addstatus if verbosity > 1: print("...after insertion:") PrintPRpoints(Result.lbz) # get the selections of new point (ie, scenarios for which delta=1) Selections = [] for scenario in ScenarioList: instance = ph._instances[scenario[0]] if getattr(instance,IndVarName).value == 1: Selections.append(scenario) Result.selections = Insert(Selections,PRpoint,Result.selections) if verbosity > 0: print("Interval "+str(PRpoint)+", ["+str(bL)+", "+str(bU)+ \ "] split at ("+str(b)+", "+str(z)+")") print("\tnew PR point has "+str(len(Selections))+" selections") if verbosity > 1: print("test that selections list aligned with lbz") if not len(Result.lbz) == len(Result.selections): print("** fatal error: lbz not= selections") PrintPRpoints(Result.lbz) print("Result.selections:") for i in range(Result.selections): print("%d %f" % (i,Result.selections[i])) return Result # ok, we have split and/or fathomed interval if NumIntervals >= maxIntervals: # we are about to leave while loop due to... addstatus = "** terminating because number of intervals = " + \ str(NumIntervals) + " >= max = " + str(maxIntervals) if verbosity > 0: print(addstatus+'\n') Result.status = Result.status + "\n" + addstatus # while loop continues if verbosity > 1: print("bottom of while loop") PrintPRpoints(Result.lbz) ################################################### # end while NumIntervals < maxIntervals: # ^ this is indentation of while loop ################ end while loop ################### if verbosity > 1: print("\nend while loop...setting multipliers") for i in range(1,len(Result.lbz)): db = Result.lbz[i][1] - Result.lbz[i-1][1] dz = Result.lbz[i][2] - Result.lbz[i-1][2] if dz > 0: Result.lbz[i][0] = dz/db else: #print "dz =",dz," at ",i,": ",Result.lbz[i]," -",Result.lbz[i-1] Result.lbz[i][0] = 0 if verbosity > 0: PrintPRpoints(Result.lbz) addstatus = '\nLagrange multiplier search ends'+datetime_string() if verbosity > 0: print(addstatus+'\n') Result.status = Result.status + addstatus outName = csvPrefix + "PRoptimal.csv" with open(outName,'w') as outFile: if verbosity > 0: print("writing PR points to "+outName+'\n') for lbz in Result.lbz: outFile.write(str(lbz[1])+ ", " +str(lbz[2])+'\n') outName = csvPrefix + "OptimalSelections.csv" with open(outName,'w') as outFile: if verbosity > 0: print("writing optimal selections for each PR point to "+csvPrefix+'PRoptimal.csv\n') for selections in Result.selections: char = "" thisSelection = "" for slist in selections: if slist: thisSelection = thisSelection + char + slist[0] char = "," outFile.write(thisSelection+'\n') if verbosity > 0: print("\nReturning status:\n %s \n=======================" % Result.status) ################################ if verbosity > 2: print("\nAbout to return...Result attributes: %d" % len(inspect.getmembers(Result))) for attr in inspect.getmembers(Result): print(attr[0]) print("\n===========================================") # LagrangeParametric ends here return Result
def run_conf( scenario_instance_factory, index_list, num_scenarios_for_solution, num_scenarios_per_sample, full_scenario_tree, xhat_ph, options, ): if options.MRP_directory_basename is None: AllInOne = True sense = xhat_ph._scenario_tree._scenarios[0]._objective_sense # in order to handle the case of scenarios that are not equally # likely, we will split the expectations for Gsupk # BUT we are going to assume that the groups themselves are # equally likely and just scale by n_g and n_g-1 for Gbar and VarG # really not always needed... # http://www.eecs.berkeley.edu/~mhoemmen/cs194/Tutorials/variance.pdf g_supk_of_xhat = [] g_bar = 0 sum_xstar_obj_given_xhat = 0 n_g = options.n_g for k in range(1, n_g + 1): gk_ph = None try: if AllInOne: start_index = num_scenarios_for_solution + (k - 1) * num_scenarios_per_sample stop_index = start_index + num_scenarios_per_sample print("") print("Computing statistics for sample k=" + str(k) + ".") if options.verbose: print("Bundle start index=" + str(start_index) + ", stop index=" + str(stop_index) + ".") # compute this xstar solution for the EF associated with # sample k. print("Loading scenario instances and initializing " "scenario tree for xstar scenario bundle.") gk_ph = ph_for_bundle( start_index, stop_index, scenario_instance_factory, full_scenario_tree, index_list, options ) else: options.instance_directory = options.MRP_directory_basename + str(k) gk_ph = PHFromScratch(options) print("Creating the xstar extensive form.") print("") print("Composite scenarios:") for scenario in gk_ph._scenario_tree._scenarios: print(scenario._name) print("") gk_ef = ExtensiveFormAlgorithm(gk_ph, options._ef_options, prefix="ef_") gk_ef.build_ef() print("Solving the xstar extensive form.") # Instance preprocessing is managed within the # ph object automatically when required for a # solve. Since we are solving the instances # outside of the ph object, we will inform it # that it should complete the instance # preprocessing early gk_ph._preprocess_scenario_instances() gk_ef.solve(io_options={"output_fixed_variable_bounds": options.write_fixed_variables}) xstar_obj = gk_ef.objective # assuming this is the absolute gap xstar_obj_gap = gk_ef.gap """ gk_ef = create_ef_instance(gk_ph._scenario_tree, generate_weighted_cvar=options.generate_weighted_cvar, cvar_weight=options.cvar_weight, risk_alpha=options.risk_alpha) print("Solving the xstar extensive form.") # Instance preprocessing is managed within the ph object # automatically when required for a solve. Since we are # solving the instances outside of the ph object, we will # inform it that it should complete the instance preprocessing # early gk_ph._preprocess_scenario_instances() ef_results = solve_ef(gk_ef, options) # as in the computation of xhat, the following is required to form a # solution to the extensive form in the scenario tree itself. gk_ph._scenario_tree.pullScenarioSolutionsFromInstances() gk_ph._scenario_tree.snapshotSolutionFromScenarios() # extract the objective function value corresponding to the # xstar solution, along with any gap information. xstar_obj = gk_ph._scenario_tree.findRootNode().computeExpectedNodeCost() # assuming this is the absolute gap xstar_obj_gap = gk_ef.solutions[0].gap# ef_results.solution(0).gap """ print("Sample extensive form objective value=" + str(xstar_obj)) # CVARHACK: if CPLEX barfed, keep trucking and bury our head # in the sand. if type(xstar_obj_gap) is UndefinedData: xstar_obj_bound = xstar_obj # EW#print("xstar_obj_bound= "+str(xstar_obj_bound)) else: if sense == minimize: xstar_obj_bound = xstar_obj - xstar_obj_gap else: xstar_obj_bound = xstar_obj + xstar_obj_gap # EW#print("xstar_obj_bound= "+str(xstar_obj_bound)) # EW#print("xstar_obj = "+str(xstar_obj)) # EW#print("xstar_obj_gap = "+str(xstar_obj_gap)) # TBD: ADD VERBOSE OUTPUT HERE # to get f(xhat) for this sample, fix the first-stage # variables and re-solve the extensive form. note that the # fixing yields side-effects on the original gk_ef, but that # is fine as it isn't used after this point. print("Solving the extensive form given the xhat solution.") # xhat = pyomo.pysp.phboundbase.ExtractInternalNodeSolutionsforInner(xhat_ph) # # fix the first stage variables # gk_root_node = gk_ph._scenario_tree.findRootNode() # root_xhat = xhat[gk_root_node._name] root_xhat = xhat_ph._scenario_tree.findRootNode()._solution for variable_id in gk_root_node._standard_variable_ids: gk_root_node.fix_variable(variable_id, root_xhat[variable_id]) # Push fixed variable statuses on instances (or # transmit to the phsolverservers), since we are not # calling the solve method on the ph object, we # need to do this manually gk_ph._push_fix_queue_to_instances() gk_ph._preprocess_scenario_instances() gk_ef.solve(io_options={"output_fixed_variable_bounds": options.write_fixed_variables}) # ef_results = solve_ef(gk_ef, options) # we don't need the solution - just the objective value. # objective_name = "MASTER" # objective = gk_ef.find_component(objective_name) xstar_obj_given_xhat = gk_ef.objective print("Sample extensive form objective value given xhat=" + str(xstar_obj_given_xhat)) # g_supk_of_xhat.append(xstar_obj_given_xhat - xstar_obj_bound) if sense == minimize: g_supk_of_xhat.append(xstar_obj_given_xhat - xstar_obj_bound) else: g_supk_of_xhat.append(-xstar_obj_given_xhat + xstar_obj_bound) g_bar += g_supk_of_xhat[k - 1] sum_xstar_obj_given_xhat += xstar_obj_given_xhat finally: if gk_ph is not None: # we are using the PHCleanup function for # convenience, but we need to prevent it # from shutting down the scenario_instance_factory # as it is managed outside this function if gk_ph._scenario_tree._scenario_instance_factory is scenario_instance_factory: gk_ph._scenario_tree._scenario_instance_factory = None PHCleanup(gk_ph) g_bar /= n_g # second pass for variance calculation (because we like storing # the g_supk) g_var = 0.0 for k in range(0, n_g): print("g_supk_of_xhat[%d]=%12.6f" % (k + 1, g_supk_of_xhat[k])) g_var = g_var + (g_supk_of_xhat[k] - g_bar) * (g_supk_of_xhat[k] - g_bar) if n_g != 1: # sample var g_var = g_var / (n_g - 1) print("") print("Raw results:") print("g_bar= " + str(g_bar)) print("g_stddev= " + str(math.sqrt(g_var))) print("Average f(xhat)= " + str(sum_xstar_obj_given_xhat / n_g)) if n_g in t_table_values: print("") print("Results summary:") t_table_entries = t_table_values[n_g] for key in sorted(iterkeys(t_table_entries)): print( "Confidence interval width for alpha=" + str(key) + " is " + str(g_bar + (t_table_entries[key] * math.sqrt(g_var) / math.sqrt(n_g))) ) else: print( "No built-in t-table entries for " + str(n_g) + " degrees of freedom - cannot calculate confidence interval width" ) if options.write_xhat_solution: print("") print("xhat solution:") scenario_tree = xhat_ph._scenario_tree first_stage = scenario_tree._stages[0] root_node = first_stage._tree_nodes[0] for key, val in iteritems(root_node._solutions): for idx in val: if val[idx] != 0.0: print("%s %s %s" % (str(key), str(idx), str(val[idx]()))) scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) if options.append_file is not None: output_file = open(options.append_file, "a") output_file.write( "\ninstancedirectory, " + str(options.instance_directory) + ", seed, " + str(options.random_seed) + ", N, " + str(scenario_count) + ", hatn, " + str(num_scenarios_for_solution) + ", n_g, " + str(options.n_g) + ", Eoffofxhat, " + str(sum_xstar_obj_given_xhat / n_g) + ", gbar, " + str(g_bar) + ", sg, " + str(math.sqrt(g_var)) + ", objforxhat, " + str(xhat_obj) + ", n," + str(num_scenarios_per_sample) ) if n_g in t_table_values: t_table_entries = t_table_values[n_g] for key in sorted(iterkeys(t_table_entries)): output_file.write( " , alpha=" + str(key) + " , " + str(g_bar + (t_table_entries[key] * math.sqrt(g_var) / math.sqrt(n_g))) ) if options.write_xhat_solution: output_file.write(" , ") scenario_tree = xhat_ph._scenario_tree first_stage = scenario_tree._stages[0] root_node = first_stage._tree_nodes[0] for key, val in iteritems(root_node._solutions): for idx in val: if val[idx] != 0.0: output_file.write("%s %s %s" % (str(key), str(idx), str(val[idx]()))) output_file.close() print("") print("Results summary appended to file=" + options.append_file) xhat_ph.release_components()
def run_conf(scenario_instance_factory, index_list, num_scenarios_for_solution, num_scenarios_per_sample, full_scenario_tree, xhat_ph, options): if options.MRP_directory_basename is None: AllInOne = True sense = xhat_ph._scenario_tree._scenarios[0]._objective_sense # in order to handle the case of scenarios that are not equally # likely, we will split the expectations for Gsupk # BUT we are going to assume that the groups themselves are # equally likely and just scale by n_g and n_g-1 for Gbar and VarG # really not always needed... # http://www.eecs.berkeley.edu/~mhoemmen/cs194/Tutorials/variance.pdf g_supk_of_xhat = [] g_bar = 0 sum_xstar_obj_given_xhat = 0 n_g = options.n_g for k in range(1, n_g + 1): gk_ph = None try: if AllInOne: start_index = num_scenarios_for_solution + \ (k-1)*num_scenarios_per_sample stop_index = start_index + num_scenarios_per_sample print("") print("Computing statistics for sample k=" + str(k) + ".") if options.verbose: print("Bundle start index=" + str(start_index) + ", stop index=" + str(stop_index) + ".") # compute this xstar solution for the EF associated with # sample k. print("Loading scenario instances and initializing " "scenario tree for xstar scenario bundle.") gk_ph = ph_for_bundle(start_index, stop_index, scenario_instance_factory, full_scenario_tree, index_list, options) else: options.instance_directory = \ options.MRP_directory_basename+str(k) gk_ph = PHFromScratch(options) print("Creating the xstar extensive form.") print("") print("Composite scenarios:") for scenario in gk_ph._scenario_tree._scenarios: print(scenario._name) print("") gk_ef = ExtensiveFormAlgorithm(gk_ph, options._ef_options, prefix="ef_") gk_ef.build_ef() print("Solving the xstar extensive form.") # Instance preprocessing is managed within the # ph object automatically when required for a # solve. Since we are solving the instances # outside of the ph object, we will inform it # that it should complete the instance # preprocessing early gk_ph._preprocess_scenario_instances() gk_ef.solve(io_options=\ {'output_fixed_variable_bounds': options.write_fixed_variables}) xstar_obj = gk_ef.objective # assuming this is the absolute gap xstar_obj_gap = gk_ef.gap """ gk_ef = create_ef_instance(gk_ph._scenario_tree, generate_weighted_cvar=options.generate_weighted_cvar, cvar_weight=options.cvar_weight, risk_alpha=options.risk_alpha) print("Solving the xstar extensive form.") # Instance preprocessing is managed within the ph object # automatically when required for a solve. Since we are # solving the instances outside of the ph object, we will # inform it that it should complete the instance preprocessing # early gk_ph._preprocess_scenario_instances() ef_results = solve_ef(gk_ef, options) # as in the computation of xhat, the following is required to form a # solution to the extensive form in the scenario tree itself. gk_ph._scenario_tree.pullScenarioSolutionsFromInstances() gk_ph._scenario_tree.snapshotSolutionFromScenarios() # extract the objective function value corresponding to the # xstar solution, along with any gap information. xstar_obj = gk_ph._scenario_tree.findRootNode().computeExpectedNodeCost() # assuming this is the absolute gap xstar_obj_gap = gk_ef.solutions[0].gap# ef_results.solution(0).gap """ print("Sample extensive form objective value=" + str(xstar_obj)) # CVARHACK: if CPLEX barfed, keep trucking and bury our head # in the sand. if type(xstar_obj_gap) is UndefinedData: xstar_obj_bound = xstar_obj #EW#print("xstar_obj_bound= "+str(xstar_obj_bound)) else: if sense == minimize: xstar_obj_bound = xstar_obj - xstar_obj_gap else: xstar_obj_bound = xstar_obj + xstar_obj_gap #EW#print("xstar_obj_bound= "+str(xstar_obj_bound)) #EW#print("xstar_obj = "+str(xstar_obj)) #EW#print("xstar_obj_gap = "+str(xstar_obj_gap)) # TBD: ADD VERBOSE OUTPUT HERE # to get f(xhat) for this sample, fix the first-stage # variables and re-solve the extensive form. note that the # fixing yields side-effects on the original gk_ef, but that # is fine as it isn't used after this point. print("Solving the extensive form given the xhat solution.") #xhat = pyomo.pysp.phboundbase.ExtractInternalNodeSolutionsforInner(xhat_ph) # # fix the first stage variables # gk_root_node = gk_ph._scenario_tree.findRootNode() #root_xhat = xhat[gk_root_node._name] root_xhat = xhat_ph._scenario_tree.findRootNode()._solution for variable_id in gk_root_node._standard_variable_ids: gk_root_node.fix_variable(variable_id, root_xhat[variable_id]) # Push fixed variable statuses on instances (or # transmit to the phsolverservers), since we are not # calling the solve method on the ph object, we # need to do this manually gk_ph._push_fix_queue_to_instances() gk_ph._preprocess_scenario_instances() gk_ef.solve(io_options=\ {'output_fixed_variable_bounds': options.write_fixed_variables}) #ef_results = solve_ef(gk_ef, options) # we don't need the solution - just the objective value. #objective_name = "MASTER" #objective = gk_ef.find_component(objective_name) xstar_obj_given_xhat = gk_ef.objective print("Sample extensive form objective value given xhat=" + str(xstar_obj_given_xhat)) #g_supk_of_xhat.append(xstar_obj_given_xhat - xstar_obj_bound) if sense == minimize: g_supk_of_xhat.append(xstar_obj_given_xhat - xstar_obj_bound) else: g_supk_of_xhat.append(-xstar_obj_given_xhat + xstar_obj_bound) g_bar += g_supk_of_xhat[k - 1] sum_xstar_obj_given_xhat += xstar_obj_given_xhat finally: if gk_ph is not None: # we are using the PHCleanup function for # convenience, but we need to prevent it # from shutting down the scenario_instance_factory # as it is managed outside this function if gk_ph._scenario_tree._scenario_instance_factory is \ scenario_instance_factory: gk_ph._scenario_tree._scenario_instance_factory = None PHCleanup(gk_ph) g_bar /= n_g # second pass for variance calculation (because we like storing # the g_supk) g_var = 0.0 for k in range(0, n_g): print("g_supk_of_xhat[%d]=%12.6f" % (k + 1, g_supk_of_xhat[k])) g_var = g_var + (g_supk_of_xhat[k] - g_bar) * \ (g_supk_of_xhat[k] - g_bar) if n_g != 1: # sample var g_var = g_var / (n_g - 1) print("") print("Raw results:") print("g_bar= " + str(g_bar)) print("g_stddev= " + str(math.sqrt(g_var))) print("Average f(xhat)= " + str(sum_xstar_obj_given_xhat / n_g)) if n_g in t_table_values: print("") print("Results summary:") t_table_entries = t_table_values[n_g] for key in sorted(iterkeys(t_table_entries)): print("Confidence interval width for alpha="+str(key) +" is "+str(g_bar + (t_table_entries[key] * \ math.sqrt(g_var) / \ math.sqrt(n_g)))) else: print( "No built-in t-table entries for " + str(n_g) + " degrees of freedom - cannot calculate confidence interval width") if options.write_xhat_solution: print("") print("xhat solution:") scenario_tree = xhat_ph._scenario_tree first_stage = scenario_tree._stages[0] root_node = first_stage._tree_nodes[0] for key, val in iteritems(root_node._solutions): for idx in val: if val[idx] != 0.0: print("%s %s %s" % (str(key), str(idx), str(val[idx]()))) scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) if options.append_file is not None: output_file = open(options.append_file, "a") output_file.write("\ninstancedirectory, " + str(options.instance_directory) + ", seed, " + str(options.random_seed) + ", N, " + str(scenario_count) + ", hatn, " + str(num_scenarios_for_solution) + ", n_g, " + str(options.n_g) + ", Eoffofxhat, " + str(sum_xstar_obj_given_xhat / n_g) + ", gbar, " + str(g_bar) + ", sg, " + str(math.sqrt(g_var)) + ", objforxhat, " + str(xhat_obj) + ", n," + str(num_scenarios_per_sample)) if n_g in t_table_values: t_table_entries = t_table_values[n_g] for key in sorted(iterkeys(t_table_entries)): output_file.write(" , alpha="+str(key)+" , " +str(g_bar + (t_table_entries[key] * \ math.sqrt(g_var) / \ math.sqrt(n_g)))) if options.write_xhat_solution: output_file.write(" , ") scenario_tree = xhat_ph._scenario_tree first_stage = scenario_tree._stages[0] root_node = first_stage._tree_nodes[0] for key, val in iteritems(root_node._solutions): for idx in val: if val[idx] != 0.0: output_file.write( "%s %s %s" % (str(key), str(idx), str(val[idx]()))) output_file.close() print("") print("Results summary appended to file=" + options.append_file) xhat_ph.release_components()
def partialLagrangeParametric(args=None): print("lagrangeParam begins ") blanks = " " # used for formatting print statements class Object(object): pass Result = Object() # options used IndVarName = options.indicator_var_name CCStageNum = options.stage_num alphaTol = options.alpha_tol MaxMorePR = options.MaxMorePR # option to include up to this many PR points above F^* with all delta fixed outputFilePrefix = options.outputFilePrefix # We write ScenarioList = name, probability # PRoptimal = probability, min-cost, [selections] # PRmore = probability, min-cost, [selections] # ================ sorted by probability ======================== # # These can be read to avoid re-computing points ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[ 0] # use rootnode to loop over scenarios if find_active_objective(ph._scenario_tree._scenarios[0]._instance, safety_checks=True).is_minimizing(): print("We are solving a MINIMIZATION problem.\n") else: print("We are solving a MAXIMIZATION problem.\n") # initialize ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, options.lambda_parm_name, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: ScenarioList.append((scenario._name, scenario._probability)) # sorts from min to max probability ScenarioList.sort(key=operator.itemgetter(1)) with open(outputFilePrefix + 'ScenarioList.csv', 'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0] + ", " + str(scenario[1]) + "\n") Result.ScenarioList = ScenarioList print("lambda= " + str(lambdaval) + " ...run begins " + str(len(ScenarioList)) + " scenarios") SolStat, zL = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = " + str(bL) + " > 0") return Result print("Initial cost = " + str(zL) + " for bL = " + str(bL)) lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) print("lambda= " + str(lambdaval) + " ...run begins") SolStat, zU = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1: print("** bU = " + str(bU) + " < 1") lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) Result.lbz = [[0, bL, zL], [None, bU, zU]] Result.selections = [[], ScenarioList] NumIntervals = 1 print("initial gap = " + str(1 - zL / zU) + " \n") print("End of test; this is only a test.") return Result
def LagrangeParametric(args=None): class Object(object): pass Result = Object() Result.status = 'LagrangeParam begins ' + datetime_string( ) + '...running new ph' ph = None blanks = " " # used for formatting print statements # options used betaMin = options.beta_min betaMax = options.beta_max betaTol = options.beta_tol gapTol = options.Lagrange_gap minProb = options.min_prob maxIntervals = options.max_intervals maxTime = options.max_time IndVarName = options.indicator_var_name multName = options.lambda_parm_name CCStageNum = options.stage_num csvPrefix = options.csvPrefix verbosity = options.verbosity verbosity = 2 # override for debug (= 3 to get super-debug) HGdebug = 0 # special debug (not public) # local...may become option optTol = gapTol #################################################################### STARTTIME = time.time() Result.status = "options set" if verbosity > 1: print("From LagrangeParametric, status = %s\tSTARTTIME = %s" \ % (str(getattr(Result,'status')), str(STARTTIME))) ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[ 0] # use rootnode to loop over scenarios ReferenceInstance = ph._instances[ rootnode._scenarios[0]._name] # arbitrary scenario if find_active_objective(ph._scenario_tree._scenarios[0]._instance, safety_checks=True).is_minimizing(): sense = 'min' else: sense = 'max' scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) if options.verbosity > 0: print("%s %s scenarios" % (str(sense), str(scenario_count))) # initialize Result.status = 'starting at ' + datetime_string() if verbosity > 0: print(Result.status) ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, multName, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() sumprob = 0. minprob = 1. maxprob = 0. # fixed = 0 to get PR point at b=0 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: instance = ph._instances[scenario._name] sname = scenario._name sprob = scenario._probability sumprob = sumprob + sprob minprob = min(minprob, sprob) maxprob = max(maxprob, sprob) ScenarioList.append([sname, sprob]) ScenarioList.sort( key=operator.itemgetter(1)) # sorts from min to max probability if verbosity > 0: print("probabilities sum to %f range: %f to %f" % (sumprob, minprob, maxprob)) Result.ScenarioList = ScenarioList # Write ScenarioList = name, probability in csv file sorted by probability outName = csvPrefix + 'ScenarioList.csv' print("writing to %s" % outName) with open(outName, 'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0] + ", " + str(scenario[1]) + '\n') Result.ScenarioList = ScenarioList addstatus = 'Scenario List written to ' + csvPrefix + 'ScenarioList.csv' Result.status = Result.status + '\n' + addstatus if verbosity > 0: print(addstatus) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %f" % lambdaval) SolStat, zL = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zL = %s" % str(zL)) bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = %s > 0 (all %s = 0)" % (str(bL), str(IndVarName))) return Result if verbosity > 0: print("Initial optimal obj = %s for bL = %s" % (str(zL), str(bL))) # fixed = 1 to get PR point at b=1 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %s" % str(lambdaval)) SolStat, zU = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zU = %s" % str(zU)) if not SolStat[0:2] == 'ok': print(str(SolStat[0:3]) + " is not 'ok'") addstatus = "** Solution is non-optimal...aborting" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1. - betaTol and verbosity > 0: print("** Warning: bU = %s < 1" % str(bU)) ### enumerate points in PR space (all but one scenario) # Result.lbz = [ [0,bL,zL], [None,bU,zU] ] # for scenario in rootnode._scenarios: # sname = scenario._name # instance = ph._instances[sname] # print "excluding scenario",sname # getattr(instance,IndVarName).value = 0 # print sname,"value =",getattr(instance,IndVarName).value,getattr(instance,IndVarName).fixed # SolStat, z = lagrUtil.solve_ph_code(ph, options) # b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) # print "solve ends with status =",SolStat,"(b, z) =",b,z # getattr(instance,IndVarName).value = 1 # Result.lbz.append([None,b,z]) # for t in instance.TimePeriods: # print "Global at",t,"=",instance.posGlobalLoadGenerateMismatch[t].value, \ # '-',instance.negGlobalLoadGenerateMismatch[t].value,"=",\ # instance.GlobalLoadGenerateMismatch[t].value,\ # "\tDemand =",instance.TotalDemand[t].value, ",",\ # "Reserve =",instance.ReserveRequirement[t].value # # PrintPRpoints(Result.lbz) # return Result #### end enumeration ######################################################################## if verbosity > 1: print("We have bU = %s ...about to free all %s for %d scenarios" % \ (str(bU), str(IndVarName), len(ScenarioList))) # free scenario selection variable lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) if verbosity > 1: print("\tall %s freed; elapsed time = %f" % (str(IndVarName), time.time() - STARTTIME)) # initialize with the two endpoints Result.lbz = [[0., bL, zL], [None, bU, zU]] Result.selections = [[], ScenarioList] NumIntervals = 1 if verbosity > 0: print("Initial relative Lagrangian gap = %f maxIntervals = %d" % (1 - zL / zU, maxIntervals)) if verbosity > 1: print("entering while loop %s" % datetime_string()) print("\n") ############ main loop to search intervals ############# ######################################################## while NumIntervals < maxIntervals: lapsedTime = time.time() - STARTTIME if lapsedTime > maxTime: addstatus = '** max time reached ' + str(lapsedTime) print(addstatus) Result.status = Result.status + '\n' + addstatus break if verbosity > 1: print("Top of while with %d intervals elapsed time = %f" % (NumIntervals, lapsedTime)) PrintPRpoints(Result.lbz) lambdaval = None ### loop over PR points to find first unfathomed interval to search ### for PRpoint in range(1, len(Result.lbz)): if Result.lbz[PRpoint][0] == None: # multiplier = None means interval with upper endpoint at PRpoint not fathomed bL = Result.lbz[PRpoint - 1][1] zL = Result.lbz[PRpoint - 1][2] bU = Result.lbz[PRpoint][1] zU = Result.lbz[PRpoint][2] lambdaval = (zU - zL) / (bU - bL) break ############################# # Exited from the for loop if verbosity > 1: print("exited for loop with PRpoint = %s ...lambdaval = %s" % (PRpoint, lambdaval)) if lambdaval == None: break # all intervals are fathomed if verbosity > 1: PrintPRpoints(Result.lbz) if verbosity > 0: print("Searching for b in [%s, %s] with %s = %f" % (str( round(bL, 4)), str(round(bU, 4)), multName, lambdaval)) # search interval (bL,bU) lagrUtil.Set_ParmValue(ph, multName, lambdaval) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- %s = %f" % (multName, lambdaval)) ######################################################### SolStat, Lagrangian = lagrUtil.solve_ph_code(ph, options) ######################################################### if not SolStat[0:2] == 'ok': addstatus = "** Solution status " + SolStat + " is not optimal" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) z = Lagrangian + lambdaval * b if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- Lagrangian = %f" % Lagrangian) print("\t- b = %s" % str(b)) print("\t- z = %s" % str(z)) print("\n") # We have PR point (b,z), which may be new or one of the endpoints ################################################################## ######### Begin tolerance tests ########## # Test that b is in [bL,bU] if verbosity > 1: print("\ttesting b") if b < bL - betaTol or b > bU + betaTol: addstatus = "** fatal error: probability (= " + str(b) + \ ") is outside interval, (" + str(bL) + ", " + str(bU) + ")" addstatus = addstatus + "\n\t(tolerance = " + str( betaTol) + ")" print(addstatus + '\n') Result.status = Result.status + addstatus return Result # Test that z is in [zL,zU] if verbosity > 1: print("\ttesting z") # using optTol as absolute tolerance (not relative) # ...if we reconsider, need to allow negative z-values if z < zL - optTol or z > zU + optTol: addstatus = "** fatal error: obj (= " + str(z) + \ ") is outside interval, (" + str(zL) + ", " + str(zU) + ")" print(addstatus + '\n') Result.status = Result.status + addstatus return Result # Ok, we have (b,z) in [(bL,zL), (bU,zU)], at least within tolerances oldLagrangian = zL - lambdaval * bL # ensure lambdaval set such that endpoints have same Lagrangian value # (this is probably unnecessary, but check anyway) if abs(oldLagrangian - (zU - lambdaval * bU)) > optTol * abs(oldLagrangian): addstatus = "** fatal error: Lagrangian at (bL,zL) = " + \ str(oldLagrangian) + " not= " + str(zU-lambdaval*bU) + \ "\n\t(optTol = " + str(optTol) + ")" Result.status = Result.status + addstatus return Result # no more fatal error tests...need to know if (b,z) is endpoint or new if verbosity > 1: print("No anomalies...testing if b = bL or bU") # Test if endpoint is an alternative optimum of Lagrangian # ...using optTol as *relative* tolerance # (could use other reference values -- eg, avg or max of old and new Lagrangian values) refValue = max(min(abs(oldLagrangian), abs(Lagrangian)), 1.) alternativeOpt = abs(oldLagrangian - Lagrangian) <= optTol * refValue # alternativeOpt = True means we computed point (b,z) is alternative optimum such that: # case 1: (b,z) = endpoint, in which case we simply fathom [bL,bU] by setting PRpoint # to [lambdaval,bU,zU] (the numeric value of multiplier means fathomed) # case 2: (b,z) is new PR point on line segment, in which case we split into # [bL,b] and [b,bU], with both fathomed if verbosity > 1: print("oldLagrangian = %s" % str(oldLagrangian)) if alternativeOpt: print(":= Lagrangian = %s" % str(Lagrangian)) else: print("> Lagrangian = %s" % str(Lagrangian)) if alternativeOpt: # setting multiplier of (bU,zU) to a numeric fathoms the interval [bL,bU] Result.lbz[PRpoint][0] = lambdaval # test if (b,z) is an endpoint newPRpoint = abs(b - bL) > betaTol and abs(b - bU) > betaTol if not newPRpoint: # ...(b,z) is NOT an endpoint (or sufficiently close), so split and fathom if verbosity > 1: print("\tnot an endpoint\tlbz = %s" % str(Result.lbz[PRpoint])) if verbosity > 0: print("Lagangian solution is new PR point on line segment of (" \ + str(bL) + ", " + str(bU) +")") print( "\tsplitting (bL,bU) into (bL,b) and (b,bU), both fathomed" ) # note: else ==> b = bL or bU, so we do nothing, having already fathomed [bL,bU] # (b,z) is new PR point, so split interval (still in while loop) ########################################## # alternative optimum ==> split & fathom: (bL,b), (b,bU) if verbosity > 1: print("\talternativeOpt %s newPRpoint = %s" % (alternativeOpt, newPRpoint)) if newPRpoint: NumIntervals += 1 if alternativeOpt: if verbosity > 1: print("\tInsert [lambdaval,b,z] at %f" % PRpoint) Result.lbz = Insert([lambdaval, b, z], PRpoint, Result.lbz) addstatus = "Added PR point on line segment of envelope" if verbosity > 0: print(addstatus + '\n') else: if verbosity > 1: print("\tInsert [None,b,z] at %f" % PRpoint) Result.lbz = Insert([None, b, z], PRpoint, Result.lbz) addstatus = "new envelope extreme point added (interval split, not fathomed)" Result.status = Result.status + "\n" + addstatus if verbosity > 1: print("...after insertion:") PrintPRpoints(Result.lbz) # get the selections of new point (ie, scenarios for which delta=1) Selections = [] for scenario in ScenarioList: instance = ph._instances[scenario[0]] if getattr(instance, IndVarName).value == 1: Selections.append(scenario) Result.selections = Insert(Selections, PRpoint, Result.selections) if verbosity > 0: print("Interval "+str(PRpoint)+", ["+str(bL)+", "+str(bU)+ \ "] split at ("+str(b)+", "+str(z)+")") print("\tnew PR point has " + str(len(Selections)) + " selections") if verbosity > 1: print("test that selections list aligned with lbz") if not len(Result.lbz) == len(Result.selections): print("** fatal error: lbz not= selections") PrintPRpoints(Result.lbz) print("Result.selections:") for i in range(Result.selections): print("%d %f" % (i, Result.selections[i])) return Result # ok, we have split and/or fathomed interval if NumIntervals >= maxIntervals: # we are about to leave while loop due to... addstatus = "** terminating because number of intervals = " + \ str(NumIntervals) + " >= max = " + str(maxIntervals) if verbosity > 0: print(addstatus + '\n') Result.status = Result.status + "\n" + addstatus # while loop continues if verbosity > 1: print("bottom of while loop") PrintPRpoints(Result.lbz) ################################################### # end while NumIntervals < maxIntervals: # ^ this is indentation of while loop ################ end while loop ################### if verbosity > 1: print("\nend while loop...setting multipliers") for i in range(1, len(Result.lbz)): db = Result.lbz[i][1] - Result.lbz[i - 1][1] dz = Result.lbz[i][2] - Result.lbz[i - 1][2] if dz > 0: Result.lbz[i][0] = dz / db else: #print "dz =",dz," at ",i,": ",Result.lbz[i]," -",Result.lbz[i-1] Result.lbz[i][0] = 0 if verbosity > 0: PrintPRpoints(Result.lbz) addstatus = '\nLagrange multiplier search ends' + datetime_string() if verbosity > 0: print(addstatus + '\n') Result.status = Result.status + addstatus outName = csvPrefix + "PRoptimal.csv" with open(outName, 'w') as outFile: if verbosity > 0: print("writing PR points to " + outName + '\n') for lbz in Result.lbz: outFile.write(str(lbz[1]) + ", " + str(lbz[2]) + '\n') outName = csvPrefix + "OptimalSelections.csv" with open(outName, 'w') as outFile: if verbosity > 0: print("writing optimal selections for each PR point to " + csvPrefix + 'PRoptimal.csv\n') for selections in Result.selections: char = "" thisSelection = "" for slist in selections: if slist: thisSelection = thisSelection + char + slist[0] char = "," outFile.write(thisSelection + '\n') if verbosity > 0: print("\nReturning status:\n %s \n=======================" % Result.status) ################################ if verbosity > 2: print("\nAbout to return...Result attributes: %d" % len(inspect.getmembers(Result))) for attr in inspect.getmembers(Result): print(attr[0]) print("\n===========================================") # LagrangeParametric ends here return Result
def LagrangeMorePR(args=None): print("lagrangeMorePR begins %s" % datetime_string()) blanks = " " # used for formatting print statements class Object(object): pass Result = Object() # options used betaTol = options.beta_tol # tolerance used to separate b-values IndVarName = options.indicator_var_name multName = options.lambda_parm_name CCStageNum = options.stage_num MaxMorePR = options.max_number # max PR points to be generated (above F^* with all delta fixed) MaxTime = options.max_time # max time before terminate csvPrefix = options.csvPrefix # input filename prefix (eg, case name) probFileName = options.probFileName # name of file containing probabilities ##HG override # options.verbosity = 2 verbosity = options.verbosity Result.status = 'starting '+datetime_string() STARTTIME = time.time() ph = PHFromScratch(options) rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing(): print("We are solving a MINIMIZATION problem.") else: print("We are solving a MAXIMIZATION problem.") # initialize ScenarioList = [] with open(csvPrefix+"ScenarioList.csv",'r') as inputFile: for line in inputFile.readlines(): L = line.split(',') ScenarioList.append([L[0],float(L[1])]) addstatus = str(len(ScenarioList))+' scenarios read from file: ' + csvPrefix+'ScenarioList.csv' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus PRoptimal = [] with open(csvPrefix+"PRoptimal.csv",'r') as inputFile: for line in inputFile.readlines(): bzS = line.split(',') PRoptimal.append( [None, float(bzS[0]), float(bzS[1])] ) addstatus = str(len(PRoptimal))+' PR points read from file: '+ csvPrefix+'PRoptimal.csv (envelope function)' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus # ensure PR points on envelope function are sorted by probability PRoptimal.sort(key=operator.itemgetter(1)) PRoptimal[0][0] = 0 # initial lambda (for b=0) for p in range(1,len(PRoptimal)): dz = PRoptimal[p][2] - PRoptimal[p-1][2] db = PRoptimal[p][1] - PRoptimal[p-1][1] PRoptimal[p][0] = dz/db if verbosity > 0: PrintPRpoints(PRoptimal) Result.PRoptimal = PRoptimal lambdaval = 0. lagrUtil.Set_ParmValue(ph, options.lambda_parm_name,lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() ## read scenarios to select for each PR point on envelope function with open(csvPrefix+"OptimalSelections.csv",'r') as inputFile: OptimalSelections = [] for line in inputFile.readlines(): if len(line) == 0: break # eof selections = line.split(',') L = len(selections) Ls = len(selections[L-1]) selections[L-1] = selections[L-1][0:Ls-1] if verbosity > 1: print(str(selections)) OptimalSelections.append(selections) Result.OptimalSelections = OptimalSelections addstatus = str(len(OptimalSelections)) + ' Optimal selections read from file: ' \ + csvPrefix + 'OptimalSelections.csv' Result.status = Result.status + '\n' + addstatus if len(OptimalSelections) == len(PRoptimal): if verbosity > 0: print(addstatus) else: addstatus = addstatus + '\n** Number of selections not equal to number of PR points' print(addstatus) Result.status = Result.status + '\n' + addstatus print(str(OptimalSelections)) print((PRoptimal)) return Result ##################################################################################### # get probabilities if probFileName is None: # ...generate from widest gap regions PRlist = FindPRpoints(options, PRoptimal) else: # ...read probabilities probList = [] with open(probFileName,'r') as inputFile: if verbosity > 0: print("reading from probList = "+probFileName) for line in inputFile.readlines(): # 1 probability per line if len(line) == 0: break prob = float(line) probList.append(prob) if verbosity > 0: print("\t "+str(len(probList))+" probabilities") if verbosity > 1: print(str(probList)) PRlist = GetPoints(options, PRoptimal, probList) if verbosity > 1: print("PRlist:") for interval in PRlist: print(str(interval)) # We now have PRlist = [[i, b], ...], where b is in PRoptimal interval (i-1,i) addstatus = str(len(PRlist)) + ' probabilities' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus ##################################################################################### lapsedTime = time.time() - STARTTIME addstatus = 'Initialize complete...lapsed time = ' + str(lapsedTime) if verbosity > 1: print(addstatus) Result.status = Result.status + '\n' + addstatus ##################################################################################### if verbosity > 1: print("\nlooping over Intervals to generate PR points by flipping heuristic") Result.morePR = [] for interval in PRlist: lapsedTime = time.time() - STARTTIME if lapsedTime > MaxTime: addstatus = '** lapsed time = ' + str(lapsedTime) + ' > max time = ' + str(MaxTime) if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus break i = interval[0] # = PR point index b = interval[1] # = target probability to reach by flipping from upper endpoint bU = PRoptimal[i][1] # = upper endpoint bL = PRoptimal[i-1][1] # = lower endpoint if verbosity > 1: print( "target probability = "+str(b)+" < bU = PRoptimal[" + str(i) + "][1]" \ " and > bL = PRoptimal["+str(i-1)+"][1]") if b < bL or b > bU: addstatus = '** probability = '+str(b) + ', not in gap interval: (' \ + str(bL) + ', ' + str(bU) + ')' print(addstatus) print(str(PRoptimal)) print(str(PRlist)) Result.status = Result.status + '\n' + addstatus return Result if verbosity > 1: print( "i = "+str(i)+" : Starting with bU = "+str(bU)+" having "+ \ str(len(OptimalSelections[i]))+ " selections:") print(str(OptimalSelections[i])) # first fix all scenarios = 0 for sname, sprob in ScenarioList: scenario = ph._scenario_tree.get_scenario(sname) lagrUtil.FixIndicatorVariableOneScenario(ph, scenario, IndVarName, 0) # now fix optimal selections = 1 for sname in OptimalSelections[i]: scenario = ph._scenario_tree.get_scenario(sname) lagrUtil.FixIndicatorVariableOneScenario(ph, scenario, IndVarName, 1) # flip scenario selections from bU until we reach b (target probability) bNew = bU for sname, sprob in ScenarioList: scenario = ph._scenario_tree.get_scenario(sname) if bNew - sprob < b: continue instance = ph._instances[sname] if getattr(instance, IndVarName).value == 0: continue bNew = bNew - sprob # flipped scenario selection lagrUtil.FixIndicatorVariableOneScenario(ph, scenario, IndVarName, 0) if verbosity > 1: print("\tflipped "+sname+" with prob = "+str(sprob)+" ...bNew = "+str(bNew)) if verbosity > 1: print("\tflipped selections reach "+str(bNew)+" >= target = "+str(b)+" (bL = "+str(bL)+")") if bNew <= bL + betaTol or bNew >= bU - betaTol: if verbosity > 0: print("\tNot generating PR point...flipping from bU failed") continue # to next interval in list # ready to solve to get cost for fixed scenario selections associated with probability = bNew if verbosity > 1: # check that scenarios are fixed as they should be totalprob = 0. for scenario in ScenarioList: sname = scenario[0] sprob = scenario[1] instance = ph._instances[sname] print("fix "+sname+" = "+str(getattr(instance,IndVarName).value)+\ " is "+str(getattr(instance,IndVarName).fixed)+" probability = "+str(sprob)) if getattr(instance,IndVarName).value == 1: totalprob = totalprob + sprob lambdaval = getattr(instance, multName).value print("\ttotal probability = %f" % totalprob) # solve (all delta fixed); lambda=0, so z = Lagrangian if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %f" % lambdaval) SolStat, z = lagrUtil.solve_ph_code(ph, options) b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- SolStat = %s" % str(SolStat)) print("\t- b = %s" % str(b)) print("\t- z = %s" % str(z)) print("(adding to more PR points)") Result.morePR.append([None,b,z]) if verbosity > 1: PrintPRpoints(Result.morePR) ###################################################### # end loop over target probabilities with open(csvPrefix+"PRmore.csv",'w') as outFile: for point in Result.morePR: outFile.write(str(point[1])+','+str(point[2])) addstatus = str(len(Result.morePR)) + ' PR points written to file: '+ csvPrefix + 'PRmore.csv' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus addstatus = 'lapsed time = ' + putcommas(time.time() - STARTTIME) if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus return Result
def partialLagrangeParametric(args=None): print("lagrangeParam begins ") blanks = " " # used for formatting print statements class Object(object): pass Result = Object() # options used IndVarName = options.indicator_var_name CCStageNum = options.stage_num alphaTol = options.alpha_tol MaxMorePR = options.MaxMorePR # option to include up to this many PR points above F^* with all delta fixed outputFilePrefix = options.outputFilePrefix # We write ScenarioList = name, probability # PRoptimal = probability, min-cost, [selections] # PRmore = probability, min-cost, [selections] # ================ sorted by probability ======================== # # These can be read to avoid re-computing points ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[0] # use rootnode to loop over scenarios if find_active_objective(ph._scenario_tree._scenarios[0]._instance,safety_checks=True).is_minimizing(): print("We are solving a MINIMIZATION problem.\n") else: print("We are solving a MAXIMIZATION problem.\n") # initialize ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, options.lambda_parm_name, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: ScenarioList.append((scenario._name, scenario._probability)) # sorts from min to max probability ScenarioList.sort(key=operator.itemgetter(1)) with open(outputFilePrefix+'ScenarioList.csv','w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0]+ ", " +str(scenario[1])+"\n") Result.ScenarioList = ScenarioList print("lambda= "+str(lambdaval)+" ...run begins "+str(len(ScenarioList))+" scenarios") SolStat, zL = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = "+str(bL)+" > 0") return Result print("Initial cost = "+str(zL)+" for bL = "+str(bL)) lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) print("lambda= "+str(lambdaval)+" ...run begins") SolStat, zU = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1: print("** bU = "+str(bU)+" < 1") lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) Result.lbz = [ [0,bL,zL], [None,bU,zU] ] Result.selections = [[], ScenarioList] NumIntervals = 1 print("initial gap = "+str(1-zL/zU)+" \n") print("End of test; this is only a test.") return Result