def _presolve(self, *args, **kwds): """ Peform presolves. """ TempfileManager.push() self._keepfiles = kwds.pop("keepfiles", False) self._define_signal_handlers = kwds.pop('use_signal_handling', None) OptSolver._presolve(self, *args, **kwds) # # Verify that the input problems exists # for filename in self._problem_files: if not os.path.exists(filename): msg = 'Solver failed to locate input problem file: %s' raise ValueError(msg % filename) # # Create command line # self._command = self.create_command_line(self.executable(), self._problem_files) self._log_file = self._command.log_file # # The pre-cleanup is probably unncessary, but also not harmful. # if (self._log_file is not None) and \ os.path.exists(self._log_file): os.remove(self._log_file) if (self._soln_file is not None) and \ os.path.exists(self._soln_file): os.remove(self._soln_file)
def _postsolve(self): # take care of the annoying GUROBI log file in the current # directory. this approach doesn't seem overly efficient, but # python os module functions doesn't accept regular expression # directly. filename_list = os.listdir(".") for filename in filename_list: # IMPT: trap the possible exception raised by the file not # existing. this can occur in pyro environments where # > 1 workers are running GUROBI, and were started # from the same directory. these logs don't matter # anyway (we redirect everything), and are largely an # annoyance. try: if re.match('gurobi\.log', filename) != None: os.remove(filename) except OSError: pass # let the base class deal with returning results. results = ILMLicensedSystemCallSolver._postsolve(self) # finally, clean any temporary files registered with the temp file # manager, created populated *directly* by this plugin. does not # include, for example, the execution script. but does include # the warm-start file. TempfileManager.pop(remove=not self._keepfiles) return results
def tearDown(self): if os.path.exists("unknown.lp"): os.unlink("unknown.lp") TempfileManager.clear_tempfiles() if os.path.exists(os.path.join(currdir,'result.yml')): os.remove(os.path.join(currdir,'result.yml')) self.model = None
def solve(self, sp, *args, **kwds): """ Solve a stochastic program. See the 'solve' method on the base class for additional keyword documentation. Args: sp: The stochastic program to solve. keep_solver_files (bool): Retain temporary solver input and output files after the solve completes. *args: Passed to the derived solver class (see the _solve_impl method). **kwds: Passed to the derived solver class (see the _solve_impl method). Returns: A results object with information about the solution. """ self._files.clear() assert self.executable is not None keep_solver_files = kwds.pop("keep_solver_files", False) TempfileManager.push() try: return super(SPSolverShellCommand, self).solve(sp, *args, **kwds) finally: # cleanup TempfileManager.pop(remove=not keep_solver_files) if keep_solver_files: logger.info("Retaining the following solver files:\n" + pprint.pformat(self.files))
def test_open_tempfile_windows(self): TempfileManager.push() fname = TempfileManager.create_tempfile() f = open(fname) try: _orig = tempfiles.deletion_errors_are_fatal tempfiles.deletion_errors_are_fatal = True with self.assertRaisesRegex(WindowsError, ".*process cannot access the file"): TempfileManager.pop() finally: tempfiles.deletion_errors_are_fatal = _orig f.close() os.remove(fname) TempfileManager.push() fname = TempfileManager.create_tempfile() f = open(fname) log = StringIO() try: _orig = tempfiles.deletion_errors_are_fatal tempfiles.deletion_errors_are_fatal = False with LoggingIntercept(log, 'pyomo.common'): TempfileManager.pop() self.assertIn("Unable to delete temporary file", log.getvalue()) finally: tempfiles.deletion_errors_are_fatal = _orig f.close() os.remove(fname)
def tearDown(self): global tempdir TempfileManager.pop() TempfileManager.tempdir = old_tempdir if os.path.exists(tempdir): shutil.rmtree(tempdir) tempdir = None
def test_scip_solve_from_instance_options(self): # Creating a dummy scip.set file in the cwd # will cover the code that prints a warning _cwd = os.getcwd() tmpdir = TempfileManager.create_tempdir() try: os.chdir(tmpdir) open(join(tmpdir, 'scip.set'), "w").close() # Test scip solve from a pyomo instance and load the solution with LoggingIntercept() as LOG: results = self.scip.solve(self.model, suffixes=['.*'], options={"limits/softtime": 100}) self.assertRegex( LOG.getvalue().replace("\n", " "), r"A file named (.*) exists in the current working " r"directory, but SCIP options are being " r"set using a separate options file. The " r"options file \1 will be ignored.") finally: os.chdir(_cwd) # We don't want the test to care about which Scip version we are using self.model.solutions.store_to(results) results.Solution(0).Message = "Scip" results.Solver.Message = "Scip" results.Solver.Time = 0 _out = TempfileManager.create_tempfile(".txt") results.write(filename=_out, times=False, format='json') self.compare_json( _out, join(currdir, "test_scip_solve_from_instance.baseline"))
def test_add1(self): """Test explicit adding of a file that is missing""" try: TempfileManager.add_tempfile(tempdir + 'add1') self.fail("Expected IOError because file 'add1' does not exist") except IOError: pass
def _postsolve(self): # take care of the annoying (and empty) CPLEX temporary files in the current directory. # this approach doesn't seem overly efficient, but python os module functions don't # accept regular expression directly. filename_list = os.listdir(".") for filename in filename_list: # CPLEX temporary files come in two flavors - cplex.log and clone*.log. # the latter is the case for multi-processor environments. # IMPT: trap the possible exception raised by the file not existing. # this can occur in pyro environments where > 1 workers are # running CPLEX, and were started from the same directory. # these logs don't matter anyway (we redirect everything), # and are largely an annoyance. try: if re.match('cplex\.log', filename) != None: os.remove(filename) elif re.match('clone\d+\.log', filename) != None: os.remove(filename) except OSError: pass # let the base class deal with returning results. results = ILMLicensedSystemCallSolver._postsolve(self) # finally, clean any temporary files registered with the temp file # manager, created populated *directly* by this plugin. does not # include, for example, the execution script. but does include # the warm-start file. TempfileManager.pop(remove=not self._keepfiles) return results
def apply(self, *args, **kwargs): """ Run the external pico_convert utility """ if len(args) != 3: raise ConverterError("Cannot apply pico_convert with more than one filename or model") _exe = pyomo.common.Executable("pico_convert") if not _exe: raise ConverterError("The 'pico_convert' application cannot be found") pico_convert_cmd = _exe.path() target=str(args[1]) if target=="cpxlp": target="lp" # NOTE: if you have an extra "." in the suffix, the pico_convert program fails to output to the correct filename. output_filename = TempfileManager.create_tempfile(suffix = 'pico_convert.' + target) if not isinstance(args[2], str): fname= TempfileManager.create_tempfile(suffix= 'pico_convert.' +str(args[0])) args[2].write(filename=fname, format=args[1]) cmd = pico_convert_cmd +" --output="+output_filename+" "+target+" "+fname else: cmd = pico_convert_cmd +" --output="+output_filename+" "+target for item in args[2:]: if not os.path.exists(item): raise ConverterError("File "+item+" does not exist!") cmd = cmd + " "+item print("Running command: "+cmd) subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if not os.path.exists(output_filename): #pragma:nocover raise ApplicationError(\ "Problem launching 'pico_convert' to create "+output_filename) return (output_filename,),None # no variable map at the moment
def create_command_line(self, executable, problem_files): # # Not the best place to catch this, but we want to make sure # that we have done the version check somewhere # if executable not in self._known_versions: self._known_versions[executable] = self._get_version(executable) _ver = self._known_versions[executable] if not _ver or _ver < (4, 58): raise RuntimeError( "Pyomo only supports versions of GLPK since 4.58; " "found version %s. Please upgrade your installation " "of GLPK" % ('.'.join(map(str, _ver)), )) # # Define log file # if self._log_file is None: self._log_file = TempfileManager.create_tempfile( suffix='.glpk.log') # # Define solution file # self._glpfile = TempfileManager.create_tempfile(suffix='.glpk.glp') self._rawfile = TempfileManager.create_tempfile(suffix='.glpk.raw') self._soln_file = self._rawfile # # Define command line # cmd = [executable] if self._timer: cmd.insert(0, self._timer) for key in self.options: opt = self.options[key] if opt is None or (isinstance(opt, str) and opt.strip() == ''): # Handle the case for options that must be # specified without a value cmd.append("--%s" % key) else: cmd.extend(["--%s" % key, str(opt)]) if self._timelimit is not None and self._timelimit > 0.0: cmd.extend(['--tmlim', str(self._timelimit)]) cmd.extend(['--write', self._rawfile]) cmd.extend(['--wglp', self._glpfile]) if self._problem_format == ProblemFormat.cpxlp: cmd.extend(['--cpxlp', problem_files[0]]) elif self._problem_format == ProblemFormat.mps: cmd.extend(['--freemps', problem_files[0]]) elif self._problem_format == ProblemFormat.mod: cmd.extend(['--math', problem_files[0]]) for fname in problem_files[1:]: cmd.extend(['--data', fname]) return Bunch(cmd=cmd, log_file=self._log_file, env=None)
def do_setup(self, flag): TempfileManager.push() if flag: if not cplexamp_available: self.skipTest("The 'cplexamp' command is not available") self.asl = SolverFactory('asl:cplexamp') else: self.asl = SolverFactory('_mock_asl:cplexamp')
def _run_ipopt_with_stats(model, solver, max_iter=500, max_cpu_time=120): """ Run the solver (must be ipopt) and return the convergence statistics Parameters ---------- model : Pyomo model The pyomo model to be solved solver : Pyomo solver The pyomo solver to use - it must be ipopt, but with whichever options are preferred max_iter : int The maximum number of iterations to allow for ipopt max_cpu_time : int The maximum cpu time to allow for ipopt (in seconds) Returns ------- Returns a tuple with (solve status object, bool (solve successful or not), number of iters, solve time) """ # ToDo: Check that the "solver" is, in fact, IPOPT TempfileManager.push() tempfile = TempfileManager.create_tempfile(suffix='ipopt_out', text=True) opts = { 'output_file': tempfile, 'max_iter': max_iter, 'max_cpu_time': max_cpu_time } status_obj = solver.solve(model, options=opts, tee=True) solved = True if status_obj.solver.termination_condition != TerminationCondition.optimal: solved = False iters = 0 time = 0 # parse the output file to get the iteration count, solver times, etc. with open(tempfile, 'r') as f: for line in f: if line.startswith('Number of Iterations....:'): tokens = line.split() iters = int(tokens[3]) elif line.startswith( 'Total CPU secs in IPOPT (w/o function evaluations) ='): tokens = line.split() time += float(tokens[9]) elif line.startswith( 'Total CPU secs in NLP function evaluations ='): tokens = line.split() time += float(tokens[8]) TempfileManager.pop(remove=True) return status_obj, solved, iters, time
def _presolve(self, *args, **kwds): # create a context in the temporary file manager for # this plugin - is "pop"ed in the _postsolve method. TempfileManager.push() # if the first argument is a string (representing a filename), # then we don't have an instance => the solver is being applied # to a file. self._warm_start_solve = kwds.pop('warmstart', False) self._warm_start_file_name = kwds.pop('warmstart_file', None) user_warmstart = False if self._warm_start_file_name is not None: user_warmstart = True # the input argument can currently be one of two things: an # instance or a filename. if a filename is provided and a # warm-start is indicated, we go ahead and create the temporary # file - assuming that the user has already, via some external # mechanism, invoked warm_start() with a instance to create the # warm start file. if self._warm_start_solve and \ isinstance(args[0], str): # we assume the user knows what they are doing... pass elif self._warm_start_solve and \ (not isinstance(args[0], str)): # assign the name of the warm start file *before* calling # the base class presolve - the base class method ends up # creating the command line, and the warm start file-name is # (obviously) needed there. if self._warm_start_file_name is None: assert not user_warmstart self._warm_start_file_name = TempfileManager.create_tempfile( suffix='.gurobi.mst') # let the base class handle any remaining keywords/actions. ILMLicensedSystemCallSolver._presolve(self, *args, **kwds) # NB: we must let the base class presolve run first so that the # symbol_map is actually constructed! if (len(args) > 0) and (not isinstance(args[0], str)): if len(args) != 1: raise ValueError( "GUROBI _presolve method can only handle a single " "problem instance - %s were supplied" % (len(args), )) # write the warm-start file - currently only supports MIPs. # we only know how to deal with a single problem instance. if self._warm_start_solve and (not user_warmstart): start_time = time.time() self._warm_start(args[0]) end_time = time.time() if self._report_timing is True: print("Warm start write time=%.2f seconds" % (end_time - start_time))
def run_pyomo(self, cmd, root): results = root + '.jsn' TempfileManager.add_tempfile(results, exists=False) output = root + '.out' TempfileManager.add_tempfile(output, exists=False) cmd = ['pyomo', 'solve', '--solver=glpk', '--results-format=json', '--save-results=%s' % results] + cmd with open(output, 'w') as f: result = subprocess.run(cmd, stdout=f, stderr=f) return result
def solve(self, model, timer: HierarchicalTimer = None): avail = self.available() if not avail: raise PyomoException( f'Solver {self.__class__} is not available ({avail}).') if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() if timer is None: timer = HierarchicalTimer() try: TempfileManager.push() if self.config.filename is None: self._filename = TempfileManager.create_tempfile() else: self._filename = self.config.filename TempfileManager.add_tempfile(self._filename + '.lp', exists=False) TempfileManager.add_tempfile(self._filename + '.log', exists=False) timer.start('write lp file') self._writer.write(model, self._filename + '.lp', timer=timer) timer.stop('write lp file') res = self._apply_solver(timer) self._last_results_object = res if self.config.report_timing: logger.info('\n' + str(timer)) return res finally: # finally, clean any temporary files registered with the # temp file manager, created/populated *directly* by this # plugin. TempfileManager.pop(remove=not self.config.keepfiles) if not self.config.keepfiles: self._filename = None
def solve(self, model, timer: HierarchicalTimer = None): self.available(exception_flag=True) if timer is None: timer = HierarchicalTimer() try: TempfileManager.push() if self.config.filename is None: self._filename = TempfileManager.create_tempfile() else: self._filename = self.config.filename TempfileManager.add_tempfile(self._filename + '.lp', exists=False) TempfileManager.add_tempfile(self._filename + '.log', exists=False) timer.start('write lp file') self._writer.write(model, self._filename + '.lp', timer=timer) timer.stop('write lp file') res = self._apply_solver(timer) if self.config.report_timing: logger.info('\n' + str(timer)) return res finally: # finally, clean any temporary files registered with the # temp file manager, created/populated *directly* by this # plugin. TempfileManager.pop(remove=not self.config.keepfiles) if not self.config.keepfiles: self._filename = None
def setUp(self): if not scip_available: self.skipTest("The 'scipampl' command is not available") TempfileManager.push() self.scip = SolverFactory('scip', solver_io='nl') m = self.model = ConcreteModel() m.v = Var() m.o = Objective(expr=m.v) m.c = Constraint(expr=m.v >= 1)
def __enter__(self): self._cwd = os.getcwd() # Add a new context TempfileManager.push() # Create a new tempdir in this context self._tempdir = TempfileManager.create_tempdir( suffix=self._suffix, prefix=self._prefix, dir=self._dir, ) os.chdir(self._tempdir)
def do_setup(self, flag): global tmpdir tmpdir = os.getcwd() os.chdir(currdir) TempfileManager.sequential_files(0) if flag: if not cplexamp_available: self.skipTest("The 'cplexamp' command is not available") self.asl = SolverFactory('asl:cplexamp') else: self.asl = SolverFactory('_mock_asl:cplexamp')
def test_capture_output_logfile_string(self): logfile = TempfileManager.create_tempfile() self.assertTrue(isinstance(logfile, str)) try: with tee.capture_output(logfile): print('HELLO WORLD') with open(logfile, 'r') as f: result = f.read() self.assertEqual('HELLO WORLD\n', result) finally: TempfileManager.clear_tempfiles()
def test_ipopt_solve_from_nl(self): # Test ipopt solve from nl file _log = TempfileManager.create_tempfile(".test_ipopt.log") results = self.ipopt.solve(join(currdir, "sisser.pyomo.nl"), logfile=_log, suffixes=['.*']) # We don't want the test to care about which Ipopt version we are using results.Solution(0).Message = "Ipopt" results.Solver.Message = "Ipopt" _out = TempfileManager.create_tempfile(".test_ipopt.txt") results.write(filename=_out, times=False, format='json') self.compare_json(_out, join(currdir, "test_solve_from_nl.baseline"))
def do_setup(self): global tmpdir tmpdir = os.getcwd() os.chdir(currdir) TempfileManager.sequential_files(0) self.scip = SolverFactory('scip', solver_io='nl') m = self.model = ConcreteModel() m.v = Var() m.o = Objective(expr=m.v) m.c = Constraint(expr=m.v >= 1)
def _postsolve(self): # let the base class deal with returning results. results = super(CBCSHELL, self)._postsolve() # finally, clean any temporary files registered with the temp file # manager, created populated *directly* by this plugin. does not # include, for example, the execution script. but does include # the warm-start file. TempfileManager.pop(remove=not self._keepfiles) return results
def create_command_line(self, executable, problem_files): # # Define log file # if self._log_file is None: self._log_file = TempfileManager.create_tempfile( suffix='.glpk.log') # # Define solution file # self._glpfile = TempfileManager.create_tempfile(suffix='.glpk.glp') self._rawfile = TempfileManager.create_tempfile(suffix='.glpk.raw') self._soln_file = self._rawfile # # Define command line # cmd = [executable] if self._timer: cmd.insert(0, self._timer) for key in self.options: opt = self.options[key] if opt is None or (isinstance(opt, string_types) and opt.strip() == ''): # Handle the case for options that must be # specified without a value cmd.append("--%s" % key) else: cmd.extend(["--%s" % key, str(opt)]) #if isinstance(opt, basestring) and ' ' in opt: # cmd.append('--%s "%s"' % (key, str(opt))) #else: # cmd.append('--%s %s' % (key, str(opt))) if self._timelimit is not None and self._timelimit > 0.0: cmd.extend(['--tmlim', str(self._timelimit)]) cmd.extend(['--write', self._rawfile]) cmd.extend(['--wglp', self._glpfile]) if self._problem_format == ProblemFormat.cpxlp: cmd.extend(['--cpxlp', problem_files[0]]) elif self._problem_format == ProblemFormat.mps: cmd.extend(['--freemps', problem_files[0]]) elif self._problem_format == ProblemFormat.mod: cmd.extend(['--math', problem_files[0]]) for fname in problem_files[1:]: cmd.extend(['--data', fname]) return Bunch(cmd=cmd, log_file=self._log_file, env=None)
def test_solve4(self): """ Test ASL - test4.nl """ _log = TempfileManager.create_tempfile(".test_solve4.log") _out = TempfileManager.create_tempfile(".test_solve4.txt") results = self.asl.solve(join(currdir, "test4.nl"), logfile=_log, suffixes=['.*']) results.write(filename=_out, times=False, format='json') _baseline = join(currdir, "test4_asl.txt") with open(_out, 'r') as out, open(_baseline, 'r') as txt: self.assertStructuredAlmostEqual(json.load(txt), json.load(out), abstol=1e-4, allow_second_superset=True)
def _presolve(self, **kwds): warmstart_flag = kwds.pop('warmstart', False) self._keepfiles = kwds.pop('keepfiles', False) self._save_results = kwds.pop('save_results', True) self._integer_only_warmstarts = kwds.pop('integer_only_warmstarts', False) # create a context in the temporary file manager for # this plugin - is "pop"ed in the _postsolve method. TempfileManager.push() self.results = None model = self._pyomo_model # this implies we have a custom solution "parser", # preventing the OptSolver _presolve method from # creating one self._results_format = ResultsFormat.soln # use the base class _presolve to consume the # important keywords OptSolver._presolve(self, **kwds) # *********************************************************** # The following code is only needed for backwards compatability of load_solutions=False. # If we ever only want to support the load_vars, load_duals, etc. methods, then this can be deleted. if self._save_results: self._smap_id = id(self._symbol_map) if isinstance(self._pyomo_model, IBlock): # BIG HACK (see pyomo.core.kernel write function) if not hasattr(self._pyomo_model, "._symbol_maps"): setattr(self._pyomo_model, "._symbol_maps", {}) getattr(self._pyomo_model, "._symbol_maps")[self._smap_id] = self._symbol_map else: self._pyomo_model.solutions.add_symbol_map(self._symbol_map) # *********************************************************** if warmstart_flag: if self.warm_start_capable(): self._warm_start() else: raise ValueError( '{0} solver plugin is not capable of warmstart.'.format( type(self))) if self._log_file is None: self._log_file = TempfileManager.create_tempfile(suffix='.log')
def test_load_local_asl_library(self): DLL = find_GSL() if not DLL: self.skipTest("Could not find the amplgsl.dll library") LIB = 'test_pyomo_external_gsl.dll' model = ConcreteModel() model.gamma = ExternalFunction(library=LIB, function="gsl_sf_gamma") model.x = Var(initialize=3, bounds=(1e-5, None)) model.o = Objective(expr=model.gamma(model.x)) with TempfileManager.new_context() as tempfile: dname = tempfile.mkdtemp() shutil.copyfile(DLL, os.path.join(dname, LIB)) # Without changing directories, the load should fail with self.assertRaises(OSError): value(model.o) # Changing directories should pick up the library try: orig_dir = os.getcwd() os.chdir(dname) self.assertAlmostEqual(value(model.o), 2.0, 7) finally: os.chdir(orig_dir)
def generate_scenario_tree_image(options): with ScenarioTreeInstanceFactory( options.model_location, options.scenario_tree_location) as factory: scenario_tree = factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed, verbose=options.verbose) with TempfileManager.push(): tmpdotfile = TempfileManager.create_tempfile(suffix=".dot") scenario_tree.save_to_dot(tmpdotfile) os.system('dot -Tpdf -o %s %s' % (options.output_file, tmpdotfile)) print("Output Saved To: %s" % (options.output_file))
def test3_write_lp(self): """ Convert from AMPL to LP """ self.model = pyomo.opt.AmplModel(join(currdir, 'test3.mod')) _test = TempfileManager.create_tempfile(suffix='test3.lp') try: self.model.write(_test) except ApplicationError: err = sys.exc_info()[1] if pyomo.common.Executable("glpsol"): self.fail("Unexpected ApplicationError - glpsol is enabled " "but not available: '%s'" % str(err)) return except pyomo.opt.ConverterError: err = sys.exc_info()[1] if pyomo.common.Executable("glpsol"): self.fail("Unexpected ConverterError - glpsol is enabled " "but not available: '%s'" % str(err)) return _base = join(currdir, 'test3.baseline.lp') with open(_test, 'r') as run, open(_base, 'r') as baseline: for line1, line2 in zip_longest(run, baseline): for _pattern in ('Problem:', ): if line1.find(_pattern) >= 0: line1 = line1[:line1.find(_pattern) + len(_pattern)] line2 = line2[:line2.find(_pattern) + len(_pattern)] self.assertEqual(line1, line2, msg="Files %s and %s differ" % (_test, _base))