def runProgram_OMP(self, echoOutput=False, nProc=1): print("Running Command:", self._progArguments) ompEnv = dict(os.environ) ompEnv["OMP_NUM_THREADS"] = str(nProc) try: prog = subproc.Popen(self._progArguments, stdin=self.stdin, stdout=subproc.PIPE, shell=False, universal_newlines=True, cwd=self.progDirectory, env=ompEnv) if echoOutput: for line in prog.stdout: print(line) rtnCode = prog.wait() if rtnCode > 0: raise subproc.CalledProcessError(rtnCode, self._progArguments) except: print(self.name + " did not exit properly!") prog.terminate() return -1 else: print(self.name + " exited with return code 0.") prog.terminate() return 0
def run_process(*popenargs, **kwargs): """ A straight copy-paste of subprocess.run() from the CPython source to support Python versions earlier than 3.5. """ # Can't put these directly in function signature because PEP-3102 is # not a thing in Python 2 input = kwargs.pop('input', None) timeout = kwargs.pop('timeout', None) check = kwargs.pop('check', False) if input is not None: if 'stdin' in kwargs: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = subprocess.PIPE with subprocess.Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() raise subprocess.TimeoutExpired(process.args, timeout, output=stdout, stderr=stderr) except: # noqa process.kill() process.wait() raise retcode = process.poll() if check and retcode: raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) return CompletedProcess(process.args, retcode, stdout, stderr)
def runProgram(self, echoOutput=False, redirect_file=sys.stdout): with stdout_redirector(open(redirect_file, "a")): print("Running Command:", self._progArguments) try: prog = subproc.Popen(self._progArguments, stdin=self.stdin, stdout=subproc.PIPE, stderr=subproc.PIPE, shell=False, universal_newlines=True, cwd=self.progDirectory, preexec_fn=os.setsid) if echoOutput: for line in prog.stdout: print(line) rtnCode = prog.wait() if rtnCode: raise subproc.CalledProcessError(rtnCode, self._progArguments) except: print(self.name + " did not exit properly!") raise prog.terminate() return -1 else: print(self.name + " exited with return code 0.") prog.terminate() return 0
def test_subprocess_raises___oasis_exception_is_raised(self): with TemporaryDirectory() as csv_dir, TemporaryDirectory() as bin_dir: Path(os.path.join(csv_dir, 'events.csv')).touch() with patch('oasislmf.model_execution.bin.subprocess.check_call', Mock(side_effect=subprocess.CalledProcessError(1, ''))): with self.assertRaises(OasisException): csv_to_bin(csv_dir, bin_dir, il=True)
def test_health_check_failed(self, mock_popen): cmd = 'failed' # Fail due to command returning a non-0 exit status. mock_popen.side_effect = subprocess.CalledProcessError(1, cmd) shell = ShellHealthCheck(cmd, timeout_secs=30) success, msg = shell() mock_popen.assert_called_once_with(cmd, shell=True, timeout=30, preexec_fn=mock.ANY) self.assertFalse(success) self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def test_health_check_failed(self, mock_sub): timeout = 30 # Fail due to command returning a non-0 exit status. mock_sub.side_effect = subprocess.CalledProcessError(1, 'failed') cmd = 'cmd to fail' shell = ShellHealthCheck(cmd, timeout_secs=timeout) success, msg = shell() mock_sub.assert_called_once_with(['cmd', 'to', 'fail'], timeout=30) self.assertFalse(success) self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def test_health_check_failed_with_wrapper(self, mock_popen): cmd = 'failed' mock_popen.side_effect = subprocess.CalledProcessError(1, cmd) shell = ShellHealthCheck(cmd, timeout_secs=30, wrapper_fn=lambda c: 'wrapped: %s' % c) success, msg = shell() self.assertEqual( mock_popen.mock_calls, [mock.call('wrapped: %s' % cmd, shell=True, timeout=30, preexec_fn=mock.ANY)]) self.assertFalse(success) self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def runProgram_MPI(self, echoOutput=False, nProc=1, timeout=-1): # mpiArgs = ["/opt/openmpi/bin/mpirun"] mpiArgs = ["mpirun"] mpiArgs.append("-np") mpiArgs.append(str(nProc)) try: nodeFile = os.environ['PBS_NODEFILE'] mpiArgs.append("-machinefile") mpiArgs.append(str(nodeFile)) except: nodeFile = None mpiArgs = mpiArgs + self._progArguments print("Running Command:", mpiArgs) try: prog = subproc.Popen(mpiArgs, stdin=self.stdin, stdout=subproc.PIPE, stderr=subproc.PIPE, shell=False, universal_newlines=True, cwd=self.progDirectory, preexec_fn=os.setsid) if echoOutput: for line in prog.stdout: print(line) if timeout > 0: print "Time Out Value:", timeout try: # stdout, stderr = prog.check_output(timeout=timeout) rtnCode = prog.communicate(timeout=timeout) except subproc.TimeoutExpired: print self.name + " timed out!" prog.kill() else: rtnCode = prog.communicate() if echoOutput: for line in prog.stdout: print line if rtnCode > 0: raise subproc.CalledProcessError(rtnCode, self._progArguments) prog.terminate() except: print(self.name + " did not exit properly!") prog.terminate() return -1 else: print(self.name + " exited with return code 0.") prog.terminate() return 0
def _module(self, *args): # based on $MODULESHOME/init/python.py if isinstance(args[0], list): # if we're passed explicit list, unpack it args = args[0] cmd = '{}/bin/modulecmd'.format(os.environ['MODULESHOME']) proc = subprocess.Popen([cmd, 'python'] + args, stdout=subprocess.PIPE) (output, error) = proc.communicate() if proc.returncode != 0: raise subprocess.CalledProcessError(returncode=proc.returncode, cmd=' '.join([cmd, 'python'] + args), output=error) exec output
def check_call_realtime(args): """Run command with arguments and yield the output as they come. Stderr is piped into stdout. :raises subprocess.CalledProcessError: if exit code is non-zero """ p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while p.poll() is None: yield p.stdout.read() yield p.stdout.read() if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, args)
def test_health_check_failed(self, mock_popen): cmd = 'failed' wrapped_cmd = 'wrapped-failed' # Fail due to command returning a non-0 exit status. mock_popen.side_effect = subprocess.CalledProcessError( 1, wrapped_cmd, output='No file.') shell = ShellHealthCheck(raw_cmd=cmd, wrapped_cmd=wrapped_cmd, timeout_secs=30) success, msg = shell() mock_popen.assert_called_once_with(wrapped_cmd, timeout=30, preexec_fn=mock.ANY, stderr=STDOUT) self.assertFalse(success) self.assertEqual( msg, "Command 'failed' returned non-zero exit status 1 with output 'No file.'" )
def do_run(cmd): try: mutex_popen.acquire() # process = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) process = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) mutex_popen.release() output = '' for line in iter(process.stdout.readline, ''): output += line out, err = process.communicate() if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, cmd, output=output) return output except subprocess.CalledProcessError, e: if log_error: LOG.error("%s" % e.output) if retries > 0: LOG.info("INFO: Re-running command '%s'" % cmd) time.sleep(sleep) return run(cmd, cache_duration_secs, log_error, retries - 1, sleep * backoff, backoff) raise e
def local_execute(command, timeout=60, ignore_return_codes=None): """ Execute a command on local machine. Returns combined stdout and stderr if return code is 0 or included in the list 'ignore_return_codes'. Otherwise raises a subprocess32 error. """ process = subprocess32.Popen(command, universal_newlines=True, stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT) # Loop until process returns or timeout expires. start = time.time() output = "" return_code = None while time.time() < start + timeout and return_code == None: return_code = process.poll() if return_code == None: try: output += process.communicate(timeout=1)[0] except subprocess32.TimeoutExpired: pass if return_code == None: # Time ran out but the process didn't end. raise subprocess32.TimeoutExpired(cmd=command, output=output, timeout=timeout) if ignore_return_codes == None: ignore_return_codes = [] if return_code in ignore_return_codes or return_code == 0: return output else: raise subprocess32.CalledProcessError(returncode=return_code, cmd=command, output=output)
def run_test( args, # type: argparse.Namespace test, # type: Dict[str, str] test_number, # type: int total_tests, # type: int timeout # type: int ): # type: (...) -> TestResult global templock out = {} # type: Dict[str,Any] outdir = outstr = outerr = None test_command = [] # type: List[str] duration = 0.0 prefix = "" suffix = "" if sys.stderr.isatty(): prefix = "\r" else: suffix = "\n" try: process = None # type: subprocess.Popen test_command = prepare_test_command(args.tool, args.args, args.testargs, test) if test.get("short_name"): sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" % (prefix, test_number, total_tests, test.get("short_name"), test.get("doc"), suffix)) else: sys.stderr.write( "%sTest [%i/%i] %s%s\n" % (prefix, test_number, total_tests, test.get("doc"), suffix)) sys.stderr.flush() start_time = time.time() stderr = subprocess.PIPE if not args.verbose else None process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=stderr) outstr, outerr = [ var.decode('utf-8') for var in process.communicate(timeout=timeout) ] return_code = process.poll() duration = time.time() - start_time if return_code: raise subprocess.CalledProcessError(return_code, " ".join(test_command)) out = json.loads(outstr) except ValueError as err: _logger.error(str(err)) _logger.error(outstr) _logger.error(outerr) except subprocess.CalledProcessError as err: if err.returncode == UNSUPPORTED_FEATURE: return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration, args.classname) if test.get("should_fail", False): return TestResult(0, outstr, outerr, duration, args.classname) _logger.error(u"""Test failed: %s""", " ".join([quote(tc) for tc in test_command])) _logger.error(test.get("doc")) _logger.error(u"Returned non-zero") _logger.error(outerr) return TestResult(1, outstr, outerr, duration, args.classname, str(err)) except (yamlscanner.ScannerError, TypeError) as err: _logger.error(u"""Test failed: %s""", u" ".join([quote(tc) for tc in test_command])) _logger.error(outstr) _logger.error(u"Parse error %s", str(err)) _logger.error(outerr) except KeyboardInterrupt: _logger.error(u"""Test interrupted: %s""", u" ".join([quote(tc) for tc in test_command])) raise except subprocess.TimeoutExpired: _logger.error(u"""Test timed out: %s""", u" ".join([quote(tc) for tc in test_command])) _logger.error(test.get("doc")) return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out") finally: if process is not None and process.returncode is None: _logger.error(u"""Terminating lingering process""") process.terminate() for _ in range(0, 3): time.sleep(1) if process.poll() is not None: break if process.returncode is None: process.kill() fail_message = '' if test.get("should_fail", False): _logger.warning(u"""Test failed: %s""", u" ".join([quote(tc) for tc in test_command])) _logger.warning(test.get("doc")) _logger.warning(u"Returned zero but it should be non-zero") return TestResult(1, outstr, outerr, duration, args.classname) try: compare(test.get("output"), out) except CompareFail as ex: _logger.warning(u"""Test failed: %s""", u" ".join([quote(tc) for tc in test_command])) _logger.warning(test.get("doc")) _logger.warning(u"Compare failure %s", ex) fail_message = str(ex) if outdir: shutil.rmtree(outdir, True) return TestResult((1 if fail_message else 0), outstr, outerr, duration, args.classname, fail_message)
def wkhtmltopdf(pages, output=None, **kwargs): """ Converts html to PDF using http://wkhtmltopdf.org/. pages: List of file paths or URLs of the html to be converted. output: Optional output file path. If None, the output is returned. **kwargs: Passed to wkhtmltopdf via _extra_args() (See https://github.com/antialize/wkhtmltopdf/blob/master/README_WKHTMLTOPDF for acceptable args.) Kwargs is passed through as arguments. e.g.: {'footer_html': 'http://example.com/foot.html'} becomes '--footer-html http://example.com/foot.html' Where there is no value passed, use True. e.g.: {'disable_javascript': True} becomes: '--disable-javascript' To disable a default option, use None. e.g: {'quiet': None'} becomes: '' example usage: wkhtmltopdf(pages=['/tmp/example.html'], dpi=300, orientation='Landscape', disable_javascript=True) """ if isinstance(pages, six.string_types): # Support a single page. pages = [pages] if output is None: # Standard output. output = '-' has_cover = kwargs.pop('has_cover', False) # Default options: options = getattr(settings, 'WKHTMLTOPDF_CMD_OPTIONS', None) if options is None: options = {'quiet': True} else: options = copy(options) options.update(kwargs) # Force --encoding utf8 unless the user has explicitly overridden this. options.setdefault('encoding', 'utf8') env = getattr(settings, 'WKHTMLTOPDF_ENV', None) if env is not None: env = dict(os.environ, **env) cmd = 'WKHTMLTOPDF_CMD' cmd = getattr(settings, cmd, os.environ.get(cmd, 'wkhtmltopdf')) # Adding 'cover' option to add cover_file to the pdf to generate. if has_cover: pages.insert(0, 'cover') ck_args = list( chain(shlex.split(cmd), _options_to_args(**options), list(pages), [output])) ck_kwargs = {'env': env} # Handling of fileno() attr. based on https://github.com/GrahamDumpleton/mod_wsgi/issues/85 try: i = sys.stderr.fileno() ck_kwargs['stderr'] = sys.stderr except (AttributeError, IOError): # can't call fileno() on mod_wsgi stderr object pass process = subprocess.run(ck_args, stdout=subprocess.PIPE, **ck_kwargs) # type: subprocess.CompletedProcess # these error codes are actually not necessarily errors. # See - https://github.com/KnpLabs/snappy/issues/177#issuecomment-141008420 if process.returncode not in { 0, 1, 2, 'X', 'Y', 'Z', }: raise subprocess.CalledProcessError(process.returncode, process.args, output=process.stdout, stderr=process.stderr) return process.stdout
def run_shell_command(command, env=None, cwd=None, dry_run=False): """Subprocess wrapper to facilitate running shell commands. See documentation for the Python2 `subprocess <https://docs.python.org/2/library/subprocess.html>`_ module. Args: commands (list of :py:obj:`str`): List of commands to execute env (:py:obj:`dict`, optional): environment variables to set, passed to `Popen`, default `None`. cwd (:py:obj:`str`, optional): child processes' working directory, passed to `Popen`. Default is `None`, which uses parent processes' directory. Returns: :py:obj:`list` of :py:obj:`str` containing output that was written to stdout by each command. Note: this is split on newlines after the fact, so if commands give != 1 lines of output this will not map to the list of commands given. Raises: CalledProcessError: If any commands return with nonzero exit code. Stderr for that command is stored in `output` attribute. """ # shouldn't lookup on each invocation, but need abs path to bash in order # to pass as executable argument. Pass executable argument because we want # bash specifically (not default /bin/sh, and we save a bit of overhead by # starting bash directly instead of from sh.) bash_exec = find_executable('bash') if not isinstance(command, six.string_types): command = ' '.join(command) if dry_run: print('DRY_RUN: call {}'.format(command)) return proc = None pid = None retcode = 1 stderr = '' try: proc = subprocess.Popen(command, shell=True, executable=bash_exec, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, bufsize=0) pid = proc.pid (stdout, stderr) = proc.communicate() retcode = proc.returncode except Exception as exc: if proc: proc.kill() stderr = stderr + "\nCaught exception {0}({1!r})".format( type(exc).__name__, exc.args) if retcode != 0: print('run_shell_command on {} (pid {}) exit status={}:{}\n'.format( command, pid, retcode, stderr)) raise subprocess.CalledProcessError(returncode=retcode, cmd=command, output=stderr) if '\0' in stdout: return stdout.split('\0') else: return stdout.splitlines()
def run_command(command, env=None, cwd=None, timeout=0, dry_run=False): """Subprocess wrapper to facilitate running single command without starting a shell. Note: We hope to save some process overhead by not running the command in a shell, but this means the command can't use piping, quoting, environment variables, or filename globbing etc. See documentation for the Python2 `subprocess <https://docs.python.org/2/library/subprocess.html>`_ module. Args: command (list of :py:obj:`str`): List of commands to execute env (:py:obj:`dict`, optional): environment variables to set, passed to `Popen`, default `None`. cwd (:py:obj:`str`, optional): child processes' working directory, passed to `Popen`. Default is `None`, which uses parent processes' directory. timeout (:py:obj:`int`, optional): Optionally, kill the command's subprocess and raise a CalledProcessError if the command doesn't finish in `timeout` seconds. Returns: :py:obj:`list` of :py:obj:`str` containing output that was written to stdout by each command. Note: this is split on newlines after the fact. Raises: CalledProcessError: If any commands return with nonzero exit code. Stderr for that command is stored in `output` attribute. """ def _timeout_handler(signum, frame): raise TimeoutAlarm if isinstance(command, six.string_types): command = shlex.split(command) cmd_str = ' '.join(command) if dry_run: print('DRY_RUN: call {}'.format(cmd_str)) return proc = None pid = None retcode = 1 stderr = '' try: proc = subprocess.Popen(command, shell=False, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, bufsize=0) pid = proc.pid # py3 has timeout built into subprocess; this is a workaround signal.signal(signal.SIGALRM, _timeout_handler) signal.alarm(int(timeout)) (stdout, stderr) = proc.communicate() signal.alarm(0) # cancel the alarm retcode = proc.returncode except TimeoutAlarm: if proc: proc.kill() retcode = errno.ETIME stderr = stderr + "\nKilled by timeout (>{}sec).".format(timeout) except Exception as exc: if proc: proc.kill() stderr = stderr + "\nCaught exception {0}({1!r})".format( type(exc).__name__, exc.args) if retcode != 0: print('run_command on {} (pid {}) exit status={}:{}\n'.format( cmd_str, pid, retcode, stderr)) raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd_str, output=stderr) if '\0' in stdout: return stdout.split('\0') else: return stdout.splitlines()
def test_retry_failed_slurm_calls(self, proc_mock): proc_mock.side_effect = subprocess.CalledProcessError(1, ["mock"]) self.check_success_after_reset(proc_mock)
def _map_bowtie2(index_prefix, R1_fn, R2_fn, output_file_name, error_file_name='/dev/null', custom_binary=False, bam_output=False, by_name=False, reads=None, read_pairs=None, yield_mappings=False, yield_unaligned=False, **options): using_input_fifos = reads != None or read_pairs != None is_paired = R2_fn != None or read_pairs != None if reads: input_fifo_source = TemporaryFifo(name='input_fifo.fastq') elif read_pairs: input_fifo_source = PairedTemporaryFifos(name='input') else: input_fifo_source = DoNothing() if yield_unaligned: if is_paired: output_fifo_source = PairedTemporaryFifos(name='unaligned') else: output_fifo_source = TemporaryFifo(name='unaligned.fastq') elif yield_mappings: output_fifo_source = TemporaryFifo(name='output_fifo.sam') else: output_fifo_source = DoNothing() with input_fifo_source, output_fifo_source: if reads: R1_fn = input_fifo_source.file_name writer = ThreadFastqWriter(reads, R1_fn) elif read_pairs: R1_fn = input_fifo_source.R1_file_name R2_fn = input_fifo_source.R2_file_name writer = ThreadPairedFastqWriter(read_pairs, R1_fn, R2_fn) if yield_unaligned: if is_paired: unal_template_fn = output_fifo_source.file_name_template unal_R1_fn = output_fifo_source.R1_file_name unal_R2_fn = output_fifo_source.R2_file_name options['unaligned_pairs_file_name'] = unal_template_fn else: unal_R1_fn = output_fifo_source.file_name options['unaligned_reads_file_name'] = unal_R1_fn elif yield_mappings: output_file_name = output_fifo_source.file_name bowtie2_process, bowtie2_command = launch_bowtie2( index_prefix, R1_fn, R2_fn, output_file_name, error_file_name, bam_output, by_name, custom_binary, **options) if yield_unaligned: if is_paired: for read_pair in fastq.read_pairs(unal_R1_fn, unal_R2_fn): yield read_pair else: for read in fastq.reads(unal_R1_fn): yield read elif yield_mappings: sam_file = pysam.Samfile(output_file_name, 'r') yield sam_file for read in sam_file: yield read _, err_output = bowtie2_process.communicate() if bowtie2_process.returncode != 0: raise subprocess.CalledProcessError( bowtie2_process.returncode, bowtie2_command, err_output, ) if bam_output: sam.index_bam(output_file_name)