def _presolve(self, *args, **kwds): """ Peform presolves. """ TempfileManager.push() self._keepfiles = kwds.pop("keepfiles", False) self._define_signal_handlers = kwds.pop('use_signal_handling',None) OptSolver._presolve(self, *args, **kwds) # # Verify that the input problems exists # for filename in self._problem_files: if not os.path.exists(filename): msg = 'Solver failed to locate input problem file: %s' raise ValueError(msg % filename) # # Create command line # self._command = self.create_command_line( self.executable(), self._problem_files) self._log_file=self._command.log_file # # The pre-cleanup is probably unncessary, but also not harmful. # if (self._log_file is not None) and \ os.path.exists(self._log_file): os.remove(self._log_file) if (self._soln_file is not None) and \ os.path.exists(self._soln_file): os.remove(self._soln_file)
def _presolve(self, *args, **kwds): """ Peform presolves. """ TempfileManager.push() self._keepfiles = kwds.pop("keepfiles", False) OptSolver._presolve(self, *args, **kwds) # # Verify that the input problems exists # for filename in self._problem_files: if not os.path.exists(filename): msg = 'Solver failed to locate input problem file: %s' raise ValueError(msg % filename) # # Create command line # self._command = self.create_command_line( self.executable(), self._problem_files) self._log_file=self._command.log_file # # The pre-cleanup is probably unncessary, but also not harmful. # if (self._log_file is not None) and \ os.path.exists(self._log_file): os.remove(self._log_file) if (self._soln_file is not None) and \ os.path.exists(self._soln_file): os.remove(self._soln_file)
def solve(self, model, timer: HierarchicalTimer = None): self.available(exception_flag=True) if timer is None: timer = HierarchicalTimer() try: TempfileManager.push() if self.config.filename is None: self._filename = TempfileManager.create_tempfile() else: self._filename = self.config.filename TempfileManager.add_tempfile(self._filename + '.lp', exists=False) TempfileManager.add_tempfile(self._filename + '.soln', exists=False) TempfileManager.add_tempfile(self._filename + '.log', exists=False) timer.start('write lp file') self._writer.write(model, self._filename+'.lp', timer=timer) timer.stop('write lp file') res = self._apply_solver(timer) if self.config.report_timing: logger.info('\n' + str(timer)) return res finally: # finally, clean any temporary files registered with the # temp file manager, created/populated *directly* by this # plugin. TempfileManager.pop(remove=not self.config.keepfiles) if not self.config.keepfiles: self._filename = None if self.config.report_timing: print(timer)
def _presolve(self, *args, **kwds): # create a context in the temporary file manager for # this plugin - is "pop"ed in the _postsolve method. TempfileManager.push() # if the first argument is a string (representing a filename), # then we don't have an instance => the solver is being applied # to a file. self._warm_start_solve = kwds.pop('warmstart', False) self._warm_start_file_name = kwds.pop('warmstart_file', None) user_warmstart = False if self._warm_start_file_name is not None: user_warmstart = True # the input argument can currently be one of two things: an instance or a filename. # if a filename is provided and a warm-start is indicated, we go ahead and # create the temporary file - assuming that the user has already, via some external # mechanism, invoked warm_start() with a instance to create the warm start file. if self._warm_start_solve and \ isinstance(args[0], string_types): # we assume the user knows what they are doing... pass elif self._warm_start_solve and \ (not isinstance(args[0], string_types)): # assign the name of the warm start file *before* calling the base class # presolve - the base class method ends up creating the command line, # and the warm start file-name is (obviously) needed there. if self._warm_start_file_name is None: assert not user_warmstart self._warm_start_file_name = TempfileManager.\ create_tempfile(suffix = '.cbc.soln') # let the base class handle any remaining keywords/actions. # let the base class handle any remaining keywords/actions. super(CBCSHELL, self)._presolve(*args, **kwds) # NB: we must let the base class presolve run first so that the # symbol_map is actually constructed! if (len(args) > 0) and (not isinstance(args[0], string_types)): if len(args) != 1: raise ValueError( "CBCplugin _presolve method can only handle a single " "problem instance - %s were supplied" % (len(args), )) # write the warm-start file - currently only supports MIPs. # we only know how to deal with a single problem instance. if self._warm_start_solve and (not user_warmstart): start_time = time.time() self._warm_start(args[0]) end_time = time.time() if self._report_timing is True: print("Warm start write time=%.2f seconds" % (end_time - start_time))
def generate_scenario_tree_image(options): with ScenarioTreeInstanceFactory( options.model_location, options.scenario_tree_location) as factory: scenario_tree = factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed, verbose=options.verbose) with TempfileManager.push(): tmpdotfile = TempfileManager.create_tempfile(suffix=".dot") scenario_tree.save_to_dot(tmpdotfile) os.system('dot -Tpdf -o %s %s' % (options.output_file, tmpdotfile)) print("Output Saved To: %s" % (options.output_file))
def generate_scenario_tree_image(options): with ScenarioTreeInstanceFactory(options.model_location, options.scenario_tree_location) as factory: scenario_tree = factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed, verbose=options.verbose, ) with TempfileManager.push(): tmpdotfile = TempfileManager.create_tempfile(suffix=".dot") scenario_tree.save_to_dot(tmpdotfile) os.system("dot -Tpdf -o %s %s" % (options.output_file, tmpdotfile)) print("Output Saved To: %s" % (options.output_file))
def run_command(command=None, parser=None, args=None, name='unknown', data=None, options=None): """ Execute a function that processes command-line arguments and then calls a command-line driver. This function provides a generic facility for executing a command function is rather generic. This function is segregated from the driver to enable profiling of the command-line execution. Required: command: The name of a function that will be executed to perform process the command-line options with a parser object. parser: The parser object that is used by the command-line function. Optional: options: If this is not None, then ignore the args option and use this to specify command options. args: Command-line arguments that are parsed. If this value is `None`, then the arguments in `sys.argv` are used to parse the command-line. name: Specifying the name of the command-line (for error messages). data: A container of labeled data. Returned: retval: Return values from the command-line execution. errorcode: 0 if Pyomo ran successfully """ # # # Parse command-line options # # retval = None errorcode = 0 if options is None: try: if type(args) is argparse.Namespace: _options = args else: _options = parser.parse_args(args=args) # Replace the parser options object with a pyutilib.misc.Options object options = pyutilib.misc.Options() for key in dir(_options): if key[0] != '_': val = getattr(_options, key) if not isinstance(val, types.MethodType): options[key] = val except SystemExit: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return Container(retval=retval, errorcode=errorcode) # # Configure loggers # configure_loggers(options=options) # # Setup I/O redirect to a file # logfile = options.runtime.logfile if not logfile is None: pyutilib.misc.setup_redirect(logfile) # # Call the main Pyomo runner with profiling # TempfileManager.push() pcount = options.runtime.profile_count if pcount > 0: if not pstats_available: if not logfile is None: pyutilib.misc.reset_redirect() msg = "Cannot use the 'profile' option. The Python 'pstats' " \ 'package cannot be imported!' raise ValueError(msg) tfile = TempfileManager.create_tempfile(suffix=".profile") tmp = profile.runctx( command.__name__ + '(options=options,parser=parser)', command.__globals__, locals(), tfile) p = pstats.Stats(tfile).strip_dirs() p.sort_stats('time', 'cumulative') p = p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) p = p.sort_stats('cumulative', 'calls') p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) p = p.sort_stats('calls') p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) retval = tmp else: # # Call the main Pyomo runner without profiling # TempfileManager.push() try: retval = command(options=options, parser=parser) except SystemExit: err = sys.exc_info()[1] # # If debugging is enabled or the 'catch' option is specified, then # exit. Otherwise, print an "Exiting..." message. # if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors): sys.exit(0) print('Exiting %s: %s' % (name, str(err))) errorcode = err.code except Exception: err = sys.exc_info()[1] # # If debugging is enabled or the 'catch' option is specified, then # pass the exception up the chain (to pyomo_excepthook) # if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors): if not logfile is None: pyutilib.misc.reset_redirect() TempfileManager.pop(remove=not options.runtime.keep_files) raise if not options.model is None and not options.model.save_file is None: model = "model " + options.model.save_file else: model = "model" global filter_excepthook if filter_excepthook: action = "loading" else: action = "running" msg = "Unexpected exception while %s %s:\n" % (action, model) # # This handles the case where the error is propagated by a KeyError. # KeyError likes to pass raw strings that don't handle newlines # (they translate "\n" to "\\n"), as well as tacking on single # quotes at either end of the error message. This undoes all that. # errStr = str(err) if type(err) == KeyError and errStr != "None": errStr = str(err).replace(r"\n", "\n")[1:-1] logging.getLogger('pyomo.core').error(msg + errStr) errorcode = 1 if not logfile is None: pyutilib.misc.reset_redirect() if options.runtime.disable_gc: gc.enable() TempfileManager.pop(remove=not options.runtime.keep_files) return Container(retval=retval, errorcode=errorcode)
def _solve_impl(self, sp, keep_solver_files=False, output_solver_log=False, symbolic_solver_labels=False): if len(sp.scenario_tree.stages) > 2: raise ValueError("SD solver does not handle more " "than 2 time-stages") if sp.objective_sense == maximize: raise ValueError("SD solver does not yet handle " "maximization problems") TempfileManager.push() try: # # Setup the SD working directory # working_directory = TempfileManager.create_tempdir() config_filename = os.path.join(working_directory, "config.sd") sdinput_directory = os.path.join(working_directory, "sdinput", "pysp_model") sdoutput_directory = os.path.join(working_directory, "sdoutput", "pysp_model") logfile = os.path.join(working_directory, "sd.log") os.makedirs(sdinput_directory) assert os.path.exists(sdinput_directory) assert not os.path.exists(sdoutput_directory) self._write_config(config_filename) if self.get_option('single_replication'): solution_filename = os.path.join( sdoutput_directory, "pysp_model.detailed_rep_soln.out") else: solution_filename = os.path.join( sdoutput_directory, "pysp_model.detailed_soln.out") # # Create the SD input files # io_options = {'symbolic_solver_labels': symbolic_solver_labels} symbol_map = None if isinstance(sp, ImplicitSP): symbol_map = pyomo.pysp.smps.smpsutils.\ convert_implicit( sdinput_directory, "pysp_model", sp, core_format='mps', io_options=io_options) else: pyomo.pysp.smps.smpsutils.\ convert_explicit( sdinput_directory, "pysp_model", sp, core_format='mps', io_options=io_options) # # Launch SD # if keep_solver_files: print("Solver working directory: '%s'" % (working_directory)) print("Solver log file: '%s'" % (logfile)) start = time.time() rc, log = pyutilib.subprocess.run( self.get_option("sd_executable"), cwd=working_directory, stdin="pysp_model", outfile=logfile, tee=output_solver_log) stop = time.time() assert os.path.exists(sdoutput_directory) # # Parse the SD solution # xhat, results = self._read_solution(solution_filename) results.solver_time = stop - start if symbol_map is not None: # load the first stage variable solution into # the reference model for symbol, varvalue in xhat.items(): symbol_map.bySymbol[symbol]().value = varvalue else: # TODO: this is a hack for the non-implicit SP case # so that this solver can still be run using # the explicit scenario intput representation results.xhat = xhat finally: # # cleanup # TempfileManager.pop( remove=not keep_solver_files) return results
def run_command(command=None, parser=None, args=None, name='unknown', data=None, options=None): """ Execute a function that processes command-line arguments and then calls a command-line driver. This function provides a generic facility for executing a command function is rather generic. This function is segregated from the driver to enable profiling of the command-line execution. Required: command: The name of a function that will be executed to perform process the command-line options with a parser object. parser: The parser object that is used by the command-line function. Optional: options: If this is not None, then ignore the args option and use this to specify command options. args: Command-line arguments that are parsed. If this value is `None`, then the arguments in `sys.argv` are used to parse the command-line. name: Specifying the name of the command-line (for error messages). data: A container of labeled data. Returned: retval: Return values from the command-line execution. errorcode: 0 if Pyomo ran successfully """ # # # Parse command-line options # # retval = None errorcode = 0 if options is None: try: if type(args) is argparse.Namespace: _options = args else: _options = parser.parse_args(args=args) # Replace the parser options object with a pyutilib.misc.Options object options = pyutilib.misc.Options() for key in dir(_options): if key[0] != '_': val = getattr(_options, key) if not isinstance(val, types.MethodType): options[key] = val except SystemExit: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return Container(retval=retval, errorcode=errorcode) # # Configure loggers # configure_loggers(options=options) # # Setup I/O redirect to a file # logfile = options.runtime.logfile if not logfile is None: pyutilib.misc.setup_redirect(logfile) # # Call the main Pyomo runner with profiling # TempfileManager.push() pcount = options.runtime.profile_count if pcount > 0: if not pstats_available: if not logfile is None: pyutilib.misc.reset_redirect() msg = "Cannot use the 'profile' option. The Python 'pstats' " \ 'package cannot be imported!' raise ValueError(msg) tfile = TempfileManager.create_tempfile(suffix=".profile") tmp = profile.runctx( command.__name__ + '(options=options,parser=parser)', command.__globals__, locals(), tfile ) p = pstats.Stats(tfile).strip_dirs() p.sort_stats('time', 'cumulative') p = p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) p = p.sort_stats('cumulative','calls') p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) p = p.sort_stats('calls') p.print_stats(pcount) p.print_callers(pcount) p.print_callees(pcount) retval = tmp else: # # Call the main Pyomo runner without profiling # TempfileManager.push() try: retval = command(options=options, parser=parser) except SystemExit: err = sys.exc_info()[1] # # If debugging is enabled or the 'catch' option is specified, then # exit. Otherwise, print an "Exiting..." message. # if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors): sys.exit(0) print('Exiting %s: %s' % (name, str(err))) errorcode = err.code except Exception: err = sys.exc_info()[1] # # If debugging is enabled or the 'catch' option is specified, then # pass the exception up the chain (to pyomo_excepthook) # if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors): if not logfile is None: pyutilib.misc.reset_redirect() TempfileManager.pop(remove=not options.runtime.keep_files) raise if not options.model is None and not options.model.save_file is None: model = "model " + options.model.save_file else: model = "model" global filter_excepthook if filter_excepthook: action = "loading" else: action = "running" msg = "Unexpected exception while %s %s:\n" % (action, model) # # This handles the case where the error is propagated by a KeyError. # KeyError likes to pass raw strings that don't handle newlines # (they translate "\n" to "\\n"), as well as tacking on single # quotes at either end of the error message. This undoes all that. # errStr = str(err) if type(err) == KeyError and errStr != "None": errStr = str(err).replace(r"\n","\n")[1:-1] logging.getLogger('pyomo.core').error(msg+errStr) errorcode = 1 if not logfile is None: pyutilib.misc.reset_redirect() if options.runtime.disable_gc: gc.enable() TempfileManager.pop(remove=not options.runtime.keep_files) return Container(retval=retval, errorcode=errorcode)
def _solve_impl(self, sp, keep_solver_files=False, output_solver_log=False, symbolic_solver_labels=False): if len(sp.scenario_tree.stages) > 2: raise ValueError("SD solver does not handle more " "than 2 time-stages") if sp.objective_sense == maximize: raise ValueError("SD solver does not yet handle " "maximization problems") TempfileManager.push() try: # # Setup the SD working directory # working_directory = TempfileManager.create_tempdir() config_filename = os.path.join(working_directory, "config.sd") sdinput_directory = os.path.join(working_directory, "sdinput", "pysp_model") sdoutput_directory = os.path.join(working_directory, "sdoutput", "pysp_model") logfile = os.path.join(working_directory, "sd.log") os.makedirs(sdinput_directory) assert os.path.exists(sdinput_directory) assert not os.path.exists(sdoutput_directory) self._write_config(config_filename) if self.get_option('single_replication'): solution_filename = os.path.join( sdoutput_directory, "pysp_model.detailed_rep_soln.out") else: solution_filename = os.path.join( sdoutput_directory, "pysp_model.detailed_soln.out") # # Create the SD input files # io_options = {'symbolic_solver_labels': symbolic_solver_labels} symbol_map = None if isinstance(sp, ImplicitSP): symbol_map = pyomo.pysp.smps.smpsutils.\ convert_implicit( sdinput_directory, "pysp_model", sp, core_format='mps', io_options=io_options) else: pyomo.pysp.smps.smpsutils.\ convert_explicit( sdinput_directory, "pysp_model", sp, core_format='mps', io_options=io_options) # # Launch SD # if keep_solver_files: print("Solver working directory: '%s'" % (working_directory)) print("Solver log file: '%s'" % (logfile)) start = time.time() rc, log = pyutilib.subprocess.run(self.get_option("sd_executable"), cwd=working_directory, stdin="pysp_model", outfile=logfile, tee=output_solver_log) stop = time.time() assert os.path.exists(sdoutput_directory) # # Parse the SD solution # xhat, results = self._read_solution(solution_filename) results.solver_time = stop - start if symbol_map is not None: # load the first stage variable solution into # the reference model for symbol, varvalue in xhat.items(): symbol_map.bySymbol[symbol]().value = varvalue else: # TODO: this is a hack for the non-implicit SP case # so that this solver can still be run using # the explicit scenario intput representation results.xhat = xhat finally: # # cleanup # TempfileManager.pop(remove=not keep_solver_files) return results
def __init__(self, pyomo_model): """ Pyomo nonlinear program interface Parameters ---------- pyomo_model: pyomo.environ.ConcreteModel Pyomo concrete model """ TempfileManager.push() try: # get the temp file names for the nl file nl_file = TempfileManager.create_tempfile(suffix='pynumero.nl') # The current AmplInterface code only supports a single # objective function Therefore, we throw an error if there # is not one (and only one) active objective function. This # is better than adding a dummy objective that the user does # not know about (since we do not have a good place to # remove this objective later) # # TODO: extend the AmplInterface and the AslNLP to correctly # handle this # # This currently addresses issue #1217 objectives = list( pyomo_model.component_data_objects(ctype=pyo.Objective, active=True, descend_into=True)) if len(objectives) != 1: raise NotImplementedError( 'The ASL interface and PyomoNLP in PyNumero currently ' 'only support single objective problems. Deactivate ' 'any extra objectives you may have, or add a dummy ' 'objective (f(x)=0) if you have a square problem.') self._objective = objectives[0] # write the nl file for the Pyomo model and get the symbolMap fname, symbolMap = WriterFactory('nl')(pyomo_model, nl_file, lambda x: True, {}) # create component maps from vardata to idx and condata to idx self._vardata_to_idx = vdidx = ComponentMap() self._condata_to_idx = cdidx = ComponentMap() # TODO: Are these names totally consistent? for name, obj in six.iteritems(symbolMap.bySymbol): if name[0] == 'v': vdidx[obj()] = int(name[1:]) elif name[0] == 'c': cdidx[obj()] = int(name[1:]) # The NL writer advertises the external function libraries # through the PYOMO_AMPLFUNC environment variable; merge it # with any preexisting AMPLFUNC definitions amplfunc = "\n".join(val for val in ( os.environ.get('AMPLFUNC', ''), os.environ.get('PYOMO_AMPLFUNC', ''), ) if val) with CtypesEnviron(AMPLFUNC=amplfunc): super(PyomoNLP, self).__init__(nl_file) # keep pyomo model in cache self._pyomo_model = pyomo_model finally: # delete the nl file TempfileManager.pop()