def run_parallel(cmds, **kwargs):
"""
run each of cmds (with shared **kwargs) using subprocess.Popen
then wait for all of them to complete
returns a list of each command's CompletedProcess
"""
procs = []
for cmd in cmds:
if not isinstance(cmd, Sequence):
continue
procs.append(Popen(cmd, **kwargs))
for cmd in cmds:
if not isinstance(cmd, Mapping):
continue
cmd_kwargs = kwargs.copy()
cmd_kwargs.update(cmd)
procs.append(Popen(**cmd_kwargs))
complete = []
for proc in procs:
result = proc.communicate()
c = CompletedProcess(proc.args, proc.returncode)
c.stdout, c.stderr = result
complete.append(c)
return complete
def _postprocess_result(result: subprocess.CompletedProcess) -> None:
if isinstance(result.stdout, bytes):
result.stdout = result.stdout.decode('utf-8', 'ignore')
if isinstance(result.stderr, bytes):
result.stderr = result.stderr.decode('utf-8', 'ignore')
if len(result.stderr) > 10240:
result.stderr = result.stderr[:10240]
def subprocess_run_stub(*args, **kwargs):
p = CompletedProcess('DONT_CARE', 1)
p.stdout = r'''[
{
"filePath": "test_file_path.js",
"messages": [
{
"ruleId": "no-unused-vars",
"severity": 2,
"message": "'x' is assigned a value but never used.",
"line": 1,
"column": 5,
"nodeType": "Identifier",
"messageId": "unusedVar",
"endLine": 1,
"endColumn": 6
}
],
"errorCount": 1,
"fatalErrorCount": 0,
"warningCount": 0,
"fixableErrorCount": 0,
"fixableWarningCount": 0,
"source": "var x = 5\nalert( 'Hello, world!' );\n",
"usedDeprecatedRules": []
}
]'''
return p
def execute(cmd, workdir=None, timeout=60):
cmd = ["/bin/bash", "-c", cmd]
try:
response = run(
cmd,
stderr=PIPE,
stdout=PIPE,
cwd=workdir,
timeout=timeout,
universal_newlines=True,
)
except TimeoutExpired:
response = CompletedProcess(args=cmd, returncode=124, stderr="Timeout")
except:
response = CompletedProcess(args=cmd,
returncode=-1,
stderr="Internal Checker Error")
response.stdout = "" if not response.stdout else str(response.stdout)
response.returncode, response.stderr = _error_decode(response)
return response
def _run_python_script(path, args):
"""
Run the given python script without launching a new process.
Starting a python process is expensive, so when we need to run a python
script, we can save a lot of time by running it in directly in this
process. In order for this to work, though, we need to temporarily
configure this process to look like a new one:
- Replace `sys.path[0]` with the directory of the script being executed.
This allows the script to use local imports.
- Put the desired arguments in `sys.argv`.
- Set `__name__` to `'__main__'`.
- Catch `SystemExit` exceptions.
- Replace the stdout file descriptor with a pipe, so that we can read any
output generated by the script no matter how it's generated.
- Replace the stdin file descriptor with an empty pipe, so that the script
read input on stdin that's meant for us.
Returns:
code: int
stdout: bytes
Note that stderr is not captured.
"""
from runpy import run_path
from traceback import print_exc
from subprocess import CompletedProcess
argv = [str(path), *args]
dir = path.parent.resolve()
p = CompletedProcess(argv, 0)
with \
_capture_stdout() as stdout, \
_preserve_stdin(), \
_munge_sys_argv(argv), \
_munge_sys_path(dir) \
:
try:
run_path(str(path), run_name='__main__')
except Exception as err:
print_exc(file=sys.stderr)
p.returncode = 1
except SystemExit as err:
# We need to handle SystemExit as if we were the python
# interpreter. From the python documentation:
#
# If [err.code] is an integer, it specifies the system exit
# status (passed to C's exit() function); if it is None, the
# exit status is zero; if it has another type (such as a
# string), the object's value is printed and the exit status is
# one.
#
# https://docs.python.org/3/library/exceptions.html?highlight=systemexit#SystemExit
if isinstance(err.code, int):
p.returncode = err.code
elif err.code is None:
p.returncode = 0
else:
print(err.code, file=sys.stderr)
p.returncode = 1
p.stdout = stdout.getvalue()
assert isinstance(p.returncode, int), repr(p.returncode)
return p
