def _solve_impl(self, sp, output_solver_log=False, verbose=False, logfile=None, **kwds): """ Solve a stochastic program with the SchurIpopt solver. See the 'solve' method on the base class for additional keyword documentation. Args: sp: The stochastic program to solve. output_solver_log (bool): Stream the solver output during the solve. logfile: The name of the logfile to save the solver output into. verbose: Report verbose status information to aid debugging. **kwds: Passed to the DDSIP file writer as I/O options (e.g., symbolic_solver_labels=True). Returns: A results object with information about the solution. """ # # Setup the SchurIpopt working directory # problem_list_filename = "PySP_Subproblems.txt" working_directory = self._create_tempdir("workdir", dir=os.getcwd()) if logfile is None: logfile = os.path.join(working_directory, "schuripopt.log") self._add_tempfile("logfile", logfile) else: self._files["logfile"] = logfile if verbose: print("Schuripopt solver working directory: %s" % (working_directory)) print("Schuripopt solver logfile: %s" % (logfile)) # # Launch SchurIpopt from the worker processes # (assumed to be launched together using mpirun) # status = self._launch_solver( sp, working_directory, logfile=logfile, output_solver_log=output_solver_log, verbose=verbose, io_options=kwds) objective = 0.0 solver_status = set() solver_message = set() termination_condition = set() solution_status = set() if status.solve_type == "bundles": assert sp.scenario_tree.contains_bundles() assert len(status.objective) == \ len(sp.scenario_tree.bundles) for bundle in sp.scenario_tree.bundles: if objective is not None: if isinstance(status.objective[bundle.name], UndefinedData): objective = None else: objective += bundle.probability * \ status.objective[bundle.name] solver_status.add(status.solver_status[bundle.name]) solver_message.add(status.solver_message[bundle.name]) termination_condition.add(status.termination_condition[bundle.name]) if isinstance(status.solution_status[bundle.name], UndefinedData): solution_status.add(None) else: solution_status.add(status.solution_status[bundle.name]) else: assert status.solve_type == "scenarios" assert len(status.objective) == \ len(sp.scenario_tree.scenarios) for scenario in sp.scenario_tree.scenarios: if objective is not None: if isinstance(status.objective[scenario.name], UndefinedData): objective = None else: objective += scenario.probability * \ status.objective[scenario.name] solver_status.add(status.solver_status[scenario.name]) solver_message.add(status.solver_message[scenario.name]) termination_condition.add(status.termination_condition[scenario.name]) if isinstance(status.solution_status[scenario.name], UndefinedData): solution_status.add(None) else: solution_status.add(status.solution_status[scenario.name]) assert len(solver_status) == 1 assert len(solver_message) == 1 assert len(termination_condition) == 1 assert len(solution_status) == 1 results = SPSolverResults() results.objective = None results.bound = None results.status = solution_status.pop() results.solver.status = solver_status.pop() results.solver.termination_condition = termination_condition.pop() results.solver.message = solver_message.pop() results.solver.time = max(status.solve_time.values()) results.solver.pyomo_time = \ max(status.pyomo_solve_time.values()) results.xhat = None if str(results.solver.status) == "ok" and \ str(results.solver.termination_condition) == "optimal": results.objective = objective xhat = results.xhat = {} for stage in sp.scenario_tree.stages[:-1]: for node in stage.nodes: worker_name = sp.get_worker_for_scenario( node.scenarios[0].name) node_solution = sp.invoke_function_on_worker( worker_name, "EXTERNAL_collect_solution", thisfile, invocation_type=InvocationType.Single, function_args=(node.name,)) node_xhat = xhat[node.name] = {} for id_ in node_solution: node_xhat[repr(id_)] = node_solution[id_][0] return results
def _solve_impl(self, sp, output_solver_log=False, keep_solver_files=False, symbolic_solver_labels=False): """ Solve a stochastic program by building the extensive form and calling and a Pyomo solver. See the 'solve' method on the base class for additional keyword documentation. Args: sp: The stochastic program to solve. Pyro based managers are not accepted. All scenario models must be managed locally. output_solver_log (bool): Stream the solver output during the solve. keep_solver_files (bool): Retain temporary solver input and output files after the solve completes. symbolic_solver_labels (bool): Generate solver input files using human-readable symbols (makes debugging easier). Returns: A results object with information about the solution. """ if isinstance(sp, (ScenarioTreeManagerClientPyro, ScenarioTreeManagerSolverClientPyro)): raise TypeError("The EF solver does not handle " "Pyro-based scenario tree managers") orig_parents = {} if sp.scenario_tree.contains_bundles(): for scenario in sp.scenario_tree.scenarios: if scenario._instance._parent is not None: orig_parents[scenario] = scenario._instance._parent scenario._instance._parent = None assert not scenario._instance_objective.active scenario._instance_objective.activate() try: with ExtensiveFormAlgorithm(sp, self._options) as ef: ef.build_ef() ef.solve( output_solver_log=output_solver_log, keep_solver_files=keep_solver_files, symbolic_solver_labels=symbolic_solver_labels, check_status=False) finally: for scenario, parent in orig_parents.items(): scenario._instance._parent = parent assert scenario._instance_objective.active scenario._instance_objective.deactivate() results = SPSolverResults() results.objective = ef.objective results.bound = ef.bound results.status = ef.solution_status results.solver.status = ef.solver_status results.solver.termination_condition = ef.termination_condition results.solver.message = ef.termination_message results.solver.time = ef.time results.solver.pyomo_time = ef.pyomo_time results.xhat = None if ef.solution_status is not None: xhat = results.xhat = {} for stage in sp.scenario_tree.stages[:-1]: for node in stage.nodes: node_xhat = xhat[node.name] = {} reference_scenario = node.scenarios[0] node_x = reference_scenario._x[node.name] for id_ in node._variable_ids: node_xhat[id_] = node_x[id_] # TODO: stage costs #for stagenum, stage in enumerate(sp.scenario_tree.stages[:-1]): # cost_variable_name, cost_variable_index = \ # stage._cost_variable # stage_cost_obj = \ # instance.find_component(cost_variable_name)[cost_variable_index] # if not stage_cost_obj.is_expression_type(): # refvar = ComponentUID(stage_cost_obj,cuid_buffer=tmp).\ # find_component(reference_model) # refvar.value = stage_cost_obj.value # refvar.stale = stage_cost_obj.stale return results
def _read_solution(self, sp, symbols_filename, info_filename, solution_filename): """Parses a DDSIP solution file.""" # parse the symbol map symbol_map = {} with open(symbols_filename) as f: for line in f: lp_symbol, scenario_tree_id = line.strip().split() symbol_map[lp_symbol] = scenario_tree_id # # Xhat # try: xhat = {} with open(solution_filename, 'r') as f: line = f.readline() while line.strip() != "1. Best Solution": line = f.readline() line = f.readline() assert line.startswith("Variable name Value") line = f.readline() assert line.startswith("-----------------------------------") line = f.readline().strip() while line != "": line = line.split() varlabel, varsol = line xhat[symbol_map[varlabel]] = float(varsol) line = f.readline().strip() except (IOError, OSError): logger.warn("Exception encountered while parsing ddsip " "solution file '%s':\n%s'" % (solution_filename, traceback.format_exc())) xhat = None # # Objective, bound, status, etc. # results = SPSolverResults() results.solver.status_code = None results.status = None results.solver.status = None results.solver.termination_condition = None results.solver.message = None results.solver.time = None try: with open(info_filename, 'r') as f: line = f.readline() while True: if line.startswith("Total CPU time:"): break line = f.readline() if line == '': # Unexpected file format or the solve failed logger.warn("Unexpected ddsip info file format. No " "status information will be returned") return xhat, results line = f.readline().strip() while (line == "") or \ (line == "----------------------------------------------------------------------------------------"): line = f.readline().strip() if line.startswith("EEV"): results.eev = float(line.split()[1]) line = f.readline().strip() if line.startswith("VSS"): results.vss = float(line.split()[1]) line = f.readline().strip() assert line.startswith("EVPI") line = line.split() results.evpi = float(line[1]) line = f.readline().strip() assert line == "" line = f.readline().strip() assert line == "----------------------------------------------------------------------------------------" line = f.readline().strip() line = line.split() assert len(line) == 4 assert line[0] == "Status" results.solver.status_code = int(line[1]) assert line[2] == "Time" results.solver.time = float(line[3]) (results.status, results.solver.status, results.solver.termination_condition, results.solver.message) = \ _ddsip_status_map.get(results.solver.status_code, (SolutionStatus.unknown, SolverStatus.unknown, TerminationCondition.unknown, None)) line = f.readline().strip() line = line.split() assert len(line) == 6 assert line[0] == "Upper" assert line[3] == "Tree" results.tree_depth = int(line[5]) line = f.readline().strip() line = line.split() assert len(line) == 2 assert line[0] == "Nodes" results.nodes = int(line[1]) line = f.readline().strip() while line != "----------------------------------------------------------------------------------------": line = f.readline().strip() line = f.readline().strip() assert line.startswith("Best Value") results.objective = float(line.split()[2]) # NOTE: I think DDSIP refers to the "bound" as # "Lower Bound", even when the objective # is being maximized. line = f.readline().strip() assert line.startswith("Lower Bound") results.bound = float(line.split()[2]) except (IOError, OSError): logger.warn("Exception encountered while parsing ddsip " "info file '%s':\n%s'" % (info_filename, traceback.format_exc())) return xhat, results
def _solve_impl(self, sp, output_solver_log=False, keep_solver_files=False, symbolic_solver_labels=False): """ Solve a stochastic program by building the extensive form and calling and a Pyomo solver. See the 'solve' method on the base class for additional keyword documentation. Args: sp: The stochastic program to solve. Pyro based managers are not accepted. All scenario models must be managed locally. output_solver_log (bool): Stream the solver output during the solve. keep_solver_files (bool): Retain temporary solver input and output files after the solve completes. symbolic_solver_labels (bool): Generate solver input files using human-readable symbols (makes debugging easier). Returns: A results object with information about the solution. """ if isinstance(sp, (ScenarioTreeManagerClientPyro, ScenarioTreeManagerSolverClientPyro)): raise TypeError("The EF solver does not handle " "Pyro-based scenario tree managers") orig_parents = {} if sp.scenario_tree.contains_bundles(): for scenario in sp.scenario_tree.scenarios: if scenario._instance._parent is not None: orig_parents[scenario] = scenario._instance._parent scenario._instance._parent = None assert not scenario._instance_objective.active scenario._instance_objective.activate() try: with ExtensiveFormAlgorithm(sp, self._options) as ef: ef.build_ef() ef.solve(output_solver_log=output_solver_log, keep_solver_files=keep_solver_files, symbolic_solver_labels=symbolic_solver_labels, check_status=False) finally: for scenario, parent in orig_parents.items(): scenario._instance._parent = parent assert scenario._instance_objective.active scenario._instance_objective.deactivate() results = SPSolverResults() results.objective = ef.objective results.bound = ef.bound results.status = ef.solution_status results.solver.status = ef.solver_status results.solver.termination_condition = ef.termination_condition results.solver.message = ef.termination_message results.solver.time = ef.time results.solver.pyomo_time = ef.pyomo_time results.xhat = None if ef.solution_status is not None: xhat = results.xhat = {} for stage in sp.scenario_tree.stages[:-1]: for node in stage.nodes: node_xhat = xhat[node.name] = {} reference_scenario = node.scenarios[0] node_x = reference_scenario._x[node.name] for id_ in node._variable_ids: node_xhat[id_] = node_x[id_] # TODO: stage costs #for stagenum, stage in enumerate(sp.scenario_tree.stages[:-1]): # cost_variable_name, cost_variable_index = \ # stage._cost_variable # stage_cost_obj = \ # instance.find_component(cost_variable_name)[cost_variable_index] # if not stage_cost_obj.is_expression_type(): # refvar = ComponentUID(stage_cost_obj,cuid_buffer=tmp).\ # find_component(reference_model) # refvar.value = stage_cost_obj.value # refvar.stale = stage_cost_obj.stale return results
def _solve_impl(self, sp, output_solver_log=False, verbose=False, logfile=None, **kwds): """ Solve a stochastic program with the SchurIpopt solver. See the 'solve' method on the base class for additional keyword documentation. Args: sp: The stochastic program to solve. output_solver_log (bool): Stream the solver output during the solve. logfile: The name of the logfile to save the solver output into. verbose: Report verbose status information to aid debugging. **kwds: Passed to the DDSIP file writer as I/O options (e.g., symbolic_solver_labels=True). Returns: A results object with information about the solution. """ # # Setup the SchurIpopt working directory # problem_list_filename = "PySP_Subproblems.txt" working_directory = self._create_tempdir("workdir", dir=os.getcwd()) if logfile is None: logfile = os.path.join(working_directory, "schuripopt.log") self._add_tempfile("logfile", logfile) else: self._files["logfile"] = logfile if verbose: print("Schuripopt solver working directory: %s" % (working_directory)) print("Schuripopt solver logfile: %s" % (logfile)) # # Launch SchurIpopt from the worker processes # (assumed to be launched together using mpirun) # status = self._launch_solver( sp, working_directory, logfile=logfile, output_solver_log=output_solver_log, verbose=verbose, io_options=kwds) objective = 0.0 solver_status = set() solver_message = set() termination_condition = set() solution_status = set() if status.solve_type == "bundles": assert sp.scenario_tree.contains_bundles() assert len(status.objective) == \ len(sp.scenario_tree.bundles) for bundle in sp.scenario_tree.bundles: if objective is not None: if isinstance(status.objective[bundle.name], UndefinedData): objective = None else: objective += bundle.probability * \ status.objective[bundle.name] solver_status.add(status.solver_status[bundle.name]) solver_message.add(status.solver_message[bundle.name]) termination_condition.add(status.termination_condition[bundle.name]) if isinstance(status.solution_status[bundle.name], UndefinedData): solution_status.add(None) else: solution_status.add(status.solution_status[bundle.name]) else: assert status.solve_type == "scenarios" assert len(status.objective) == \ len(sp.scenario_tree.scenarios) for scenario in sp.scenario_tree.scenarios: if objective is not None: if isinstance(status.objective[scenario.name], UndefinedData): objective = None else: objective += scenario.probability * \ status.objective[scenario.name] solver_status.add(status.solver_status[scenario.name]) solver_message.add(status.solver_message[scenario.name]) termination_condition.add(status.termination_condition[scenario.name]) if isinstance(status.solution_status[scenario.name], UndefinedData): solution_status.add(None) else: solution_status.add(status.solution_status[scenario.name]) assert len(solver_status) == 1 assert len(solver_message) == 1 assert len(termination_condition) == 1 assert len(solution_status) == 1 results = SPSolverResults() results.objective = None results.bound = None results.status = solution_status.pop() results.solver.status = solver_status.pop() results.solver.termination_condition = termination_condition.pop() results.solver.message = solver_message.pop() results.solver.time = max(status.solve_time.values()) results.solver.pyomo_time = \ max(status.pyomo_solve_time.values()) results.xhat = None if str(results.solver.status) == "ok" and \ str(results.solver.termination_condition) == "optimal": results.objective = objective xhat = results.xhat = {} for stage in sp.scenario_tree.stages[:-1]: for node in stage.nodes: worker_name = sp.get_worker_for_scenario( node.scenarios[0].name) node_solution = sp.invoke_function_on_worker( worker_name, "EXTERNAL_collect_solution", thisfile, invocation_type=InvocationType.Single, function_args=(node.name,)) node_xhat = xhat[node.name] = {} for id_ in node_solution: node_xhat[repr(id_)] = node_solution[id_][0] return results
def _read_solution(self, symbols_filename, solution_filename): """ Parses an SD solution file """ # parse the symbol map symbol_map = {} with open(symbols_filename) as f: for line in f: lp_symbol, scenario_tree_id = line.strip().split() symbol_map[lp_symbol] = scenario_tree_id results = SPSolverResults() results.status = None results.solver.status = None results.solver.termination_condition = None results.solver.message = None xhat = {} try: with open(solution_filename, 'r') as f: line = f.readline() assert line.startswith("Problem:") assert line.split()[1].strip() == "pysp_model" line = f.readline() assert line.startswith("First Stage Rows:") line = f.readline() assert line.startswith("First Stage Columns:") line = f.readline() assert line.startswith("First Stage Non-zeros:") line = f.readline() assert line.startswith("Replication No.") or \ line.startswith("Number of replications:") line = f.readline() assert line.startswith("Status:") results.solver.message = line.split(":")[1].strip() (results.status, results.solver.status, results.solver.termination_condition) = \ _sd_status_map.get(results.solver.message, (SolutionStatus.unknown, SolverStatus.unknown, TerminationCondition.unknown)) # # Objective and Bound # line = f.readline() assert line.startswith("Total Objective Function Upper Bound:") line = line.split(':') if line[1].strip() == '': pass else: assert len(line) == 4 line = line[1] if "half-width" in line: # we are given confidence intervals on the objective line = line.split(',') assert len(line) == 4 results.objective = float(line[0]) assert line[1].startswith('[') assert line[2].endswith(']') results.objective_interval = (float(line[1][1:]), float(line[2][:-1])) else: results.objective = float(line[1]) line = f.readline() assert line.startswith("Total Objective Function Lower Bound:") line = line.split(':') if line[1].strip() == '': pass else: if "half-width" in line[1]: # we are given confidence intervals on the bound line = line[1].split(',') assert len(line) == 4 results.bound = float(line[0]) assert line[1].startswith('[') assert line[2].endswith(']') results.bound_interval = (float(line[1][1:]), float(line[2][:-1])) else: results.bound = float(line[1].strip()) # # Xhat # line = f.readline() assert line.strip() == '' line = f.readline() assert line.startswith('First Stage Solutions:') line = f.readline() assert line.startswith( ' No. Row name Activity Lower bound Upper bound Dual Dual STDEV' ) line = f.readline() assert line.startswith( '------ ------------ ------------- ------------- ------------- ------------- -------------' ) xhat_start_line = ' No. Column name Activity Lower bound Upper bound Reduced Cost RC STDEV' line = f.readline() while not line.startswith(xhat_start_line): line = f.readline() line = f.readline() assert line.startswith( '------ ------------ ------------- ------------- ------------- ------------- -------------' ) line = f.readline().strip().split() while line: varlabel, varvalue = line[1:3] varlabel = varlabel.strip() varvalue = float(varvalue) xhat[symbol_map[varlabel]] = varvalue line = f.readline().strip().split() except (IOError, OSError): logger.warn("Exception encountered while parsing sd " "solution file '%s':\n%s'" % (solution_filename, traceback.format_exc())) xhat = None return xhat, results
def _read_solution(self, sp, symbols_filename, info_filename, solution_filename): """Parses a DDSIP solution file.""" # parse the symbol map symbol_map = {} with open(symbols_filename) as f: for line in f: lp_symbol, scenario_tree_id = line.strip().split() symbol_map[lp_symbol] = scenario_tree_id # # Xhat # try: xhat = {} with open(solution_filename, 'r') as f: line = f.readline() while line.strip() != "1. Best Solution": line = f.readline() line = f.readline() assert line.startswith("Variable name Value") line = f.readline() assert line.startswith("-----------------------------------") line = f.readline().strip() while line != "": line = line.split() varlabel, varsol = line xhat[symbol_map[varlabel]] = float(varsol) line = f.readline().strip() except (IOError, OSError): logger.warn( "Exception encountered while parsing ddsip " "solution file '%s':\n%s'" % (solution_filename, traceback.format_exc())) xhat = None # # Objective, bound, status, etc. # results = SPSolverResults() results.solver.status_code = None results.status = None results.solver.status = None results.solver.termination_condition = None results.solver.message = None results.solver.time = None try: with open(info_filename, 'r') as f: line = f.readline() while True: if line.startswith("Total CPU time:"): break line = f.readline() if line == '': # Unexpected file format or the solve failed logger.warn( "Unexpected ddsip info file format. No " "status information will be returned") return xhat, results line = f.readline().strip() while (line == "") or \ (line == "----------------------------------------------------------------------------------------"): line = f.readline().strip() if line.startswith("EEV"): results.eev = float(line.split()[1]) line = f.readline().strip() if line.startswith("VSS"): results.vss = float(line.split()[1]) line = f.readline().strip() assert line.startswith("EVPI") line = line.split() results.evpi = float(line[1]) line = f.readline().strip() assert line == "" line = f.readline().strip() assert line == "----------------------------------------------------------------------------------------" line = f.readline().strip() line = line.split() assert len(line) == 4 assert line[0] == "Status" results.solver.status_code = int(line[1]) assert line[2] == "Time" results.solver.time = float(line[3]) (results.status, results.solver.status, results.solver.termination_condition, results.solver.message) = \ _ddsip_status_map.get(results.solver.status_code, (SolutionStatus.unknown, SolverStatus.unknown, TerminationCondition.unknown, None)) line = f.readline().strip() line = line.split() assert len(line) == 6 assert line[0] == "Upper" assert line[3] == "Tree" results.tree_depth = int(line[5]) line = f.readline().strip() line = line.split() assert len(line) == 2 assert line[0] == "Nodes" results.nodes = int(line[1]) line = f.readline().strip() while line != "----------------------------------------------------------------------------------------": line = f.readline().strip() line = f.readline().strip() assert line.startswith("Best Value") results.objective = float(line.split()[2]) # NOTE: I think DDSIP refers to the "bound" as # "Lower Bound", even when the objective # is being maximized. line = f.readline().strip() assert line.startswith("Lower Bound") results.bound = float(line.split()[2]) except (IOError, OSError): logger.warn( "Exception encountered while parsing ddsip " "info file '%s':\n%s'" % (info_filename, traceback.format_exc())) return xhat, results