def runProgram_OMP(self, echoOutput=False, nProc=1):
print("Running Command:", self._progArguments)
ompEnv = dict(os.environ)
ompEnv["OMP_NUM_THREADS"] = str(nProc)
try:
prog = subproc.Popen(self._progArguments,
stdin=self.stdin,
stdout=subproc.PIPE,
shell=False,
universal_newlines=True,
cwd=self.progDirectory,
env=ompEnv)
if echoOutput:
for line in prog.stdout:
print(line)
rtnCode = prog.wait()
if rtnCode > 0:
raise subproc.CalledProcessError(rtnCode, self._progArguments)
except:
print(self.name + " did not exit properly!")
prog.terminate()
return -1
else:
print(self.name + " exited with return code 0.")
prog.terminate()
return 0
def run_process(*popenargs, **kwargs):
"""
A straight copy-paste of subprocess.run() from the CPython source to
support Python versions earlier than 3.5.
"""
# Can't put these directly in function signature because PEP-3102 is
# not a thing in Python 2
input = kwargs.pop('input', None)
timeout = kwargs.pop('timeout', None)
check = kwargs.pop('check', False)
if input is not None:
if 'stdin' in kwargs:
raise ValueError('stdin and input arguments may not both be used.')
kwargs['stdin'] = subprocess.PIPE
with subprocess.Popen(*popenargs, **kwargs) as process:
try:
stdout, stderr = process.communicate(input, timeout=timeout)
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
raise subprocess.TimeoutExpired(process.args, timeout,
output=stdout, stderr=stderr)
except: # noqa
process.kill()
process.wait()
raise
retcode = process.poll()
if check and retcode:
raise subprocess.CalledProcessError(retcode, process.args,
output=stdout, stderr=stderr)
return CompletedProcess(process.args, retcode, stdout, stderr)
def runProgram(self, echoOutput=False, redirect_file=sys.stdout):
with stdout_redirector(open(redirect_file, "a")):
print("Running Command:", self._progArguments)
try:
prog = subproc.Popen(self._progArguments,
stdin=self.stdin,
stdout=subproc.PIPE,
stderr=subproc.PIPE,
shell=False,
universal_newlines=True,
cwd=self.progDirectory,
preexec_fn=os.setsid)
if echoOutput:
for line in prog.stdout:
print(line)
rtnCode = prog.wait()
if rtnCode:
raise subproc.CalledProcessError(rtnCode,
self._progArguments)
except:
print(self.name + " did not exit properly!")
raise
prog.terminate()
return -1
else:
print(self.name + " exited with return code 0.")
prog.terminate()
return 0
def test_subprocess_raises___oasis_exception_is_raised(self):
with TemporaryDirectory() as csv_dir, TemporaryDirectory() as bin_dir:
Path(os.path.join(csv_dir, 'events.csv')).touch()
with patch('oasislmf.model_execution.bin.subprocess.check_call',
Mock(side_effect=subprocess.CalledProcessError(1, ''))):
with self.assertRaises(OasisException):
csv_to_bin(csv_dir, bin_dir, il=True)
def test_health_check_failed(self, mock_popen):
cmd = 'failed'
# Fail due to command returning a non-0 exit status.
mock_popen.side_effect = subprocess.CalledProcessError(1, cmd)
shell = ShellHealthCheck(cmd, timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with(cmd, shell=True, timeout=30, preexec_fn=mock.ANY)
self.assertFalse(success)
self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def test_health_check_failed(self, mock_sub):
timeout = 30
# Fail due to command returning a non-0 exit status.
mock_sub.side_effect = subprocess.CalledProcessError(1, 'failed')
cmd = 'cmd to fail'
shell = ShellHealthCheck(cmd, timeout_secs=timeout)
success, msg = shell()
mock_sub.assert_called_once_with(['cmd', 'to', 'fail'], timeout=30)
self.assertFalse(success)
self.assertEqual(msg,
"Command 'failed' returned non-zero exit status 1")
def test_health_check_failed_with_wrapper(self, mock_popen):
cmd = 'failed'
mock_popen.side_effect = subprocess.CalledProcessError(1, cmd)
shell = ShellHealthCheck(cmd, timeout_secs=30, wrapper_fn=lambda c: 'wrapped: %s' % c)
success, msg = shell()
self.assertEqual(
mock_popen.mock_calls,
[mock.call('wrapped: %s' % cmd, shell=True, timeout=30, preexec_fn=mock.ANY)])
self.assertFalse(success)
self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def runProgram_MPI(self, echoOutput=False, nProc=1, timeout=-1):
# mpiArgs = ["/opt/openmpi/bin/mpirun"]
mpiArgs = ["mpirun"]
mpiArgs.append("-np")
mpiArgs.append(str(nProc))
try:
nodeFile = os.environ['PBS_NODEFILE']
mpiArgs.append("-machinefile")
mpiArgs.append(str(nodeFile))
except:
nodeFile = None
mpiArgs = mpiArgs + self._progArguments
print("Running Command:", mpiArgs)
try:
prog = subproc.Popen(mpiArgs,
stdin=self.stdin,
stdout=subproc.PIPE,
stderr=subproc.PIPE,
shell=False,
universal_newlines=True,
cwd=self.progDirectory,
preexec_fn=os.setsid)
if echoOutput:
for line in prog.stdout:
print(line)
if timeout > 0:
print "Time Out Value:", timeout
try:
# stdout, stderr = prog.check_output(timeout=timeout)
rtnCode = prog.communicate(timeout=timeout)
except subproc.TimeoutExpired:
print self.name + " timed out!"
prog.kill()
else:
rtnCode = prog.communicate()
if echoOutput:
for line in prog.stdout:
print line
if rtnCode > 0:
raise subproc.CalledProcessError(rtnCode, self._progArguments)
prog.terminate()
except:
print(self.name + " did not exit properly!")
prog.terminate()
return -1
else:
print(self.name + " exited with return code 0.")
prog.terminate()
return 0
def _module(self, *args):
# based on $MODULESHOME/init/python.py
if isinstance(args[0],
list): # if we're passed explicit list, unpack it
args = args[0]
cmd = '{}/bin/modulecmd'.format(os.environ['MODULESHOME'])
proc = subprocess.Popen([cmd, 'python'] + args, stdout=subprocess.PIPE)
(output, error) = proc.communicate()
if proc.returncode != 0:
raise subprocess.CalledProcessError(returncode=proc.returncode,
cmd=' '.join([cmd, 'python'] +
args),
output=error)
exec output
def check_call_realtime(args):
"""Run command with arguments and yield the output as they come.
Stderr is piped into stdout.
:raises subprocess.CalledProcessError: if exit code is non-zero
"""
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while p.poll() is None:
yield p.stdout.read()
yield p.stdout.read()
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, args)
def test_health_check_failed(self, mock_popen):
cmd = 'failed'
wrapped_cmd = 'wrapped-failed'
# Fail due to command returning a non-0 exit status.
mock_popen.side_effect = subprocess.CalledProcessError(
1, wrapped_cmd, output='No file.')
shell = ShellHealthCheck(raw_cmd=cmd,
wrapped_cmd=wrapped_cmd,
timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with(wrapped_cmd,
timeout=30,
preexec_fn=mock.ANY,
stderr=STDOUT)
self.assertFalse(success)
self.assertEqual(
msg,
"Command 'failed' returned non-zero exit status 1 with output 'No file.'"
)
def do_run(cmd):
try:
mutex_popen.acquire()
# process = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
process = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
mutex_popen.release()
output = ''
for line in iter(process.stdout.readline, ''):
output += line
out, err = process.communicate()
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, cmd, output=output)
return output
except subprocess.CalledProcessError, e:
if log_error:
LOG.error("%s" % e.output)
if retries > 0:
LOG.info("INFO: Re-running command '%s'" % cmd)
time.sleep(sleep)
return run(cmd, cache_duration_secs, log_error, retries - 1, sleep * backoff, backoff)
raise e
def local_execute(command, timeout=60, ignore_return_codes=None):
"""
Execute a command on local machine. Returns combined stdout and stderr if
return code is 0 or included in the list 'ignore_return_codes'. Otherwise
raises a subprocess32 error.
"""
process = subprocess32.Popen(command,
universal_newlines=True,
stdout=subprocess32.PIPE,
stderr=subprocess32.STDOUT)
# Loop until process returns or timeout expires.
start = time.time()
output = ""
return_code = None
while time.time() < start + timeout and return_code == None:
return_code = process.poll()
if return_code == None:
try:
output += process.communicate(timeout=1)[0]
except subprocess32.TimeoutExpired:
pass
if return_code == None:
# Time ran out but the process didn't end.
raise subprocess32.TimeoutExpired(cmd=command,
output=output,
timeout=timeout)
if ignore_return_codes == None:
ignore_return_codes = []
if return_code in ignore_return_codes or return_code == 0:
return output
else:
raise subprocess32.CalledProcessError(returncode=return_code,
cmd=command,
output=output)
def run_test(
args, # type: argparse.Namespace
test, # type: Dict[str, str]
test_number, # type: int
total_tests, # type: int
timeout # type: int
): # type: (...) -> TestResult
global templock
out = {} # type: Dict[str,Any]
outdir = outstr = outerr = None
test_command = [] # type: List[str]
duration = 0.0
prefix = ""
suffix = ""
if sys.stderr.isatty():
prefix = "\r"
else:
suffix = "\n"
try:
process = None # type: subprocess.Popen
test_command = prepare_test_command(args.tool, args.args,
args.testargs, test)
if test.get("short_name"):
sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" %
(prefix, test_number, total_tests,
test.get("short_name"), test.get("doc"), suffix))
else:
sys.stderr.write(
"%sTest [%i/%i] %s%s\n" %
(prefix, test_number, total_tests, test.get("doc"), suffix))
sys.stderr.flush()
start_time = time.time()
stderr = subprocess.PIPE if not args.verbose else None
process = subprocess.Popen(test_command,
stdout=subprocess.PIPE,
stderr=stderr)
outstr, outerr = [
var.decode('utf-8') for var in process.communicate(timeout=timeout)
]
return_code = process.poll()
duration = time.time() - start_time
if return_code:
raise subprocess.CalledProcessError(return_code,
" ".join(test_command))
out = json.loads(outstr)
except ValueError as err:
_logger.error(str(err))
_logger.error(outstr)
_logger.error(outerr)
except subprocess.CalledProcessError as err:
if err.returncode == UNSUPPORTED_FEATURE:
return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration,
args.classname)
if test.get("should_fail", False):
return TestResult(0, outstr, outerr, duration, args.classname)
_logger.error(u"""Test failed: %s""",
" ".join([quote(tc) for tc in test_command]))
_logger.error(test.get("doc"))
_logger.error(u"Returned non-zero")
_logger.error(outerr)
return TestResult(1, outstr, outerr, duration, args.classname,
str(err))
except (yamlscanner.ScannerError, TypeError) as err:
_logger.error(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.error(outstr)
_logger.error(u"Parse error %s", str(err))
_logger.error(outerr)
except KeyboardInterrupt:
_logger.error(u"""Test interrupted: %s""",
u" ".join([quote(tc) for tc in test_command]))
raise
except subprocess.TimeoutExpired:
_logger.error(u"""Test timed out: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.error(test.get("doc"))
return TestResult(2, outstr, outerr, timeout, args.classname,
"Test timed out")
finally:
if process is not None and process.returncode is None:
_logger.error(u"""Terminating lingering process""")
process.terminate()
for _ in range(0, 3):
time.sleep(1)
if process.poll() is not None:
break
if process.returncode is None:
process.kill()
fail_message = ''
if test.get("should_fail", False):
_logger.warning(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.warning(test.get("doc"))
_logger.warning(u"Returned zero but it should be non-zero")
return TestResult(1, outstr, outerr, duration, args.classname)
try:
compare(test.get("output"), out)
except CompareFail as ex:
_logger.warning(u"""Test failed: %s""",
u" ".join([quote(tc) for tc in test_command]))
_logger.warning(test.get("doc"))
_logger.warning(u"Compare failure %s", ex)
fail_message = str(ex)
if outdir:
shutil.rmtree(outdir, True)
return TestResult((1 if fail_message else 0), outstr, outerr, duration,
args.classname, fail_message)
def wkhtmltopdf(pages, output=None, **kwargs):
"""
Converts html to PDF using http://wkhtmltopdf.org/.
pages: List of file paths or URLs of the html to be converted.
output: Optional output file path. If None, the output is returned.
**kwargs: Passed to wkhtmltopdf via _extra_args() (See
https://github.com/antialize/wkhtmltopdf/blob/master/README_WKHTMLTOPDF
for acceptable args.)
Kwargs is passed through as arguments. e.g.:
{'footer_html': 'http://example.com/foot.html'}
becomes
'--footer-html http://example.com/foot.html'
Where there is no value passed, use True. e.g.:
{'disable_javascript': True}
becomes:
'--disable-javascript'
To disable a default option, use None. e.g:
{'quiet': None'}
becomes:
''
example usage:
wkhtmltopdf(pages=['/tmp/example.html'],
dpi=300,
orientation='Landscape',
disable_javascript=True)
"""
if isinstance(pages, six.string_types):
# Support a single page.
pages = [pages]
if output is None:
# Standard output.
output = '-'
has_cover = kwargs.pop('has_cover', False)
# Default options:
options = getattr(settings, 'WKHTMLTOPDF_CMD_OPTIONS', None)
if options is None:
options = {'quiet': True}
else:
options = copy(options)
options.update(kwargs)
# Force --encoding utf8 unless the user has explicitly overridden this.
options.setdefault('encoding', 'utf8')
env = getattr(settings, 'WKHTMLTOPDF_ENV', None)
if env is not None:
env = dict(os.environ, **env)
cmd = 'WKHTMLTOPDF_CMD'
cmd = getattr(settings, cmd, os.environ.get(cmd, 'wkhtmltopdf'))
# Adding 'cover' option to add cover_file to the pdf to generate.
if has_cover:
pages.insert(0, 'cover')
ck_args = list(
chain(shlex.split(cmd), _options_to_args(**options), list(pages),
[output]))
ck_kwargs = {'env': env}
# Handling of fileno() attr. based on https://github.com/GrahamDumpleton/mod_wsgi/issues/85
try:
i = sys.stderr.fileno()
ck_kwargs['stderr'] = sys.stderr
except (AttributeError, IOError):
# can't call fileno() on mod_wsgi stderr object
pass
process = subprocess.run(ck_args, stdout=subprocess.PIPE,
**ck_kwargs) # type: subprocess.CompletedProcess
# these error codes are actually not necessarily errors.
# See - https://github.com/KnpLabs/snappy/issues/177#issuecomment-141008420
if process.returncode not in {
0,
1,
2,
'X',
'Y',
'Z',
}:
raise subprocess.CalledProcessError(process.returncode,
process.args,
output=process.stdout,
stderr=process.stderr)
return process.stdout
def run_shell_command(command, env=None, cwd=None, dry_run=False):
"""Subprocess wrapper to facilitate running shell commands.
See documentation for the Python2 `subprocess
<https://docs.python.org/2/library/subprocess.html>`_ module.
Args:
commands (list of :py:obj:`str`): List of commands to execute
env (:py:obj:`dict`, optional): environment variables to set, passed to
`Popen`, default `None`.
cwd (:py:obj:`str`, optional): child processes' working directory, passed
to `Popen`. Default is `None`, which uses parent processes' directory.
Returns:
:py:obj:`list` of :py:obj:`str` containing output that was written to stdout
by each command. Note: this is split on newlines after the fact, so if
commands give != 1 lines of output this will not map to the list of commands
given.
Raises:
CalledProcessError: If any commands return with nonzero exit code.
Stderr for that command is stored in `output` attribute.
"""
# shouldn't lookup on each invocation, but need abs path to bash in order
# to pass as executable argument. Pass executable argument because we want
# bash specifically (not default /bin/sh, and we save a bit of overhead by
# starting bash directly instead of from sh.)
bash_exec = find_executable('bash')
if not isinstance(command, six.string_types):
command = ' '.join(command)
if dry_run:
print('DRY_RUN: call {}'.format(command))
return
proc = None
pid = None
retcode = 1
stderr = ''
try:
proc = subprocess.Popen(command,
shell=True,
executable=bash_exec,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=0)
pid = proc.pid
(stdout, stderr) = proc.communicate()
retcode = proc.returncode
except Exception as exc:
if proc:
proc.kill()
stderr = stderr + "\nCaught exception {0}({1!r})".format(
type(exc).__name__, exc.args)
if retcode != 0:
print('run_shell_command on {} (pid {}) exit status={}:{}\n'.format(
command, pid, retcode, stderr))
raise subprocess.CalledProcessError(returncode=retcode,
cmd=command,
output=stderr)
if '\0' in stdout:
return stdout.split('\0')
else:
return stdout.splitlines()
def run_command(command, env=None, cwd=None, timeout=0, dry_run=False):
"""Subprocess wrapper to facilitate running single command without starting
a shell.
Note:
We hope to save some process overhead by not running the command in a
shell, but this means the command can't use piping, quoting, environment
variables, or filename globbing etc.
See documentation for the Python2 `subprocess
<https://docs.python.org/2/library/subprocess.html>`_ module.
Args:
command (list of :py:obj:`str`): List of commands to execute
env (:py:obj:`dict`, optional): environment variables to set, passed to
`Popen`, default `None`.
cwd (:py:obj:`str`, optional): child processes' working directory, passed
to `Popen`. Default is `None`, which uses parent processes' directory.
timeout (:py:obj:`int`, optional): Optionally, kill the command's subprocess
and raise a CalledProcessError if the command doesn't finish in
`timeout` seconds.
Returns:
:py:obj:`list` of :py:obj:`str` containing output that was written to stdout
by each command. Note: this is split on newlines after the fact.
Raises:
CalledProcessError: If any commands return with nonzero exit code.
Stderr for that command is stored in `output` attribute.
"""
def _timeout_handler(signum, frame):
raise TimeoutAlarm
if isinstance(command, six.string_types):
command = shlex.split(command)
cmd_str = ' '.join(command)
if dry_run:
print('DRY_RUN: call {}'.format(cmd_str))
return
proc = None
pid = None
retcode = 1
stderr = ''
try:
proc = subprocess.Popen(command,
shell=False,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=0)
pid = proc.pid
# py3 has timeout built into subprocess; this is a workaround
signal.signal(signal.SIGALRM, _timeout_handler)
signal.alarm(int(timeout))
(stdout, stderr) = proc.communicate()
signal.alarm(0) # cancel the alarm
retcode = proc.returncode
except TimeoutAlarm:
if proc:
proc.kill()
retcode = errno.ETIME
stderr = stderr + "\nKilled by timeout (>{}sec).".format(timeout)
except Exception as exc:
if proc:
proc.kill()
stderr = stderr + "\nCaught exception {0}({1!r})".format(
type(exc).__name__, exc.args)
if retcode != 0:
print('run_command on {} (pid {}) exit status={}:{}\n'.format(
cmd_str, pid, retcode, stderr))
raise subprocess.CalledProcessError(returncode=retcode,
cmd=cmd_str,
output=stderr)
if '\0' in stdout:
return stdout.split('\0')
else:
return stdout.splitlines()
def test_retry_failed_slurm_calls(self, proc_mock):
proc_mock.side_effect = subprocess.CalledProcessError(1, ["mock"])
self.check_success_after_reset(proc_mock)
def _map_bowtie2(index_prefix,
R1_fn,
R2_fn,
output_file_name,
error_file_name='/dev/null',
custom_binary=False,
bam_output=False,
by_name=False,
reads=None,
read_pairs=None,
yield_mappings=False,
yield_unaligned=False,
**options):
using_input_fifos = reads != None or read_pairs != None
is_paired = R2_fn != None or read_pairs != None
if reads:
input_fifo_source = TemporaryFifo(name='input_fifo.fastq')
elif read_pairs:
input_fifo_source = PairedTemporaryFifos(name='input')
else:
input_fifo_source = DoNothing()
if yield_unaligned:
if is_paired:
output_fifo_source = PairedTemporaryFifos(name='unaligned')
else:
output_fifo_source = TemporaryFifo(name='unaligned.fastq')
elif yield_mappings:
output_fifo_source = TemporaryFifo(name='output_fifo.sam')
else:
output_fifo_source = DoNothing()
with input_fifo_source, output_fifo_source:
if reads:
R1_fn = input_fifo_source.file_name
writer = ThreadFastqWriter(reads, R1_fn)
elif read_pairs:
R1_fn = input_fifo_source.R1_file_name
R2_fn = input_fifo_source.R2_file_name
writer = ThreadPairedFastqWriter(read_pairs, R1_fn, R2_fn)
if yield_unaligned:
if is_paired:
unal_template_fn = output_fifo_source.file_name_template
unal_R1_fn = output_fifo_source.R1_file_name
unal_R2_fn = output_fifo_source.R2_file_name
options['unaligned_pairs_file_name'] = unal_template_fn
else:
unal_R1_fn = output_fifo_source.file_name
options['unaligned_reads_file_name'] = unal_R1_fn
elif yield_mappings:
output_file_name = output_fifo_source.file_name
bowtie2_process, bowtie2_command = launch_bowtie2(
index_prefix, R1_fn, R2_fn, output_file_name, error_file_name,
bam_output, by_name, custom_binary, **options)
if yield_unaligned:
if is_paired:
for read_pair in fastq.read_pairs(unal_R1_fn, unal_R2_fn):
yield read_pair
else:
for read in fastq.reads(unal_R1_fn):
yield read
elif yield_mappings:
sam_file = pysam.Samfile(output_file_name, 'r')
yield sam_file
for read in sam_file:
yield read
_, err_output = bowtie2_process.communicate()
if bowtie2_process.returncode != 0:
raise subprocess.CalledProcessError(
bowtie2_process.returncode,
bowtie2_command,
err_output,
)
if bam_output:
sam.index_bam(output_file_name)
