def run_parallel(cmds, **kwargs): """ run each of cmds (with shared **kwargs) using subprocess.Popen then wait for all of them to complete returns a list of each command's CompletedProcess """ procs = [] for cmd in cmds: if not isinstance(cmd, Sequence): continue procs.append(Popen(cmd, **kwargs)) for cmd in cmds: if not isinstance(cmd, Mapping): continue cmd_kwargs = kwargs.copy() cmd_kwargs.update(cmd) procs.append(Popen(**cmd_kwargs)) complete = [] for proc in procs: result = proc.communicate() c = CompletedProcess(proc.args, proc.returncode) c.stdout, c.stderr = result complete.append(c) return complete
def _postprocess_result(result: subprocess.CompletedProcess) -> None: if isinstance(result.stdout, bytes): result.stdout = result.stdout.decode('utf-8', 'ignore') if isinstance(result.stderr, bytes): result.stderr = result.stderr.decode('utf-8', 'ignore') if len(result.stderr) > 10240: result.stderr = result.stderr[:10240]
def subprocess_run_stub(*args, **kwargs): p = CompletedProcess('DONT_CARE', 1) p.stdout = r'''[ { "filePath": "test_file_path.js", "messages": [ { "ruleId": "no-unused-vars", "severity": 2, "message": "'x' is assigned a value but never used.", "line": 1, "column": 5, "nodeType": "Identifier", "messageId": "unusedVar", "endLine": 1, "endColumn": 6 } ], "errorCount": 1, "fatalErrorCount": 0, "warningCount": 0, "fixableErrorCount": 0, "fixableWarningCount": 0, "source": "var x = 5\nalert( 'Hello, world!' );\n", "usedDeprecatedRules": [] } ]''' return p
def execute(cmd, workdir=None, timeout=60): cmd = ["/bin/bash", "-c", cmd] try: response = run( cmd, stderr=PIPE, stdout=PIPE, cwd=workdir, timeout=timeout, universal_newlines=True, ) except TimeoutExpired: response = CompletedProcess(args=cmd, returncode=124, stderr="Timeout") except: response = CompletedProcess(args=cmd, returncode=-1, stderr="Internal Checker Error") response.stdout = "" if not response.stdout else str(response.stdout) response.returncode, response.stderr = _error_decode(response) return response
def _run_python_script(path, args): """ Run the given python script without launching a new process. Starting a python process is expensive, so when we need to run a python script, we can save a lot of time by running it in directly in this process. In order for this to work, though, we need to temporarily configure this process to look like a new one: - Replace `sys.path[0]` with the directory of the script being executed. This allows the script to use local imports. - Put the desired arguments in `sys.argv`. - Set `__name__` to `'__main__'`. - Catch `SystemExit` exceptions. - Replace the stdout file descriptor with a pipe, so that we can read any output generated by the script no matter how it's generated. - Replace the stdin file descriptor with an empty pipe, so that the script read input on stdin that's meant for us. Returns: code: int stdout: bytes Note that stderr is not captured. """ from runpy import run_path from traceback import print_exc from subprocess import CompletedProcess argv = [str(path), *args] dir = path.parent.resolve() p = CompletedProcess(argv, 0) with \ _capture_stdout() as stdout, \ _preserve_stdin(), \ _munge_sys_argv(argv), \ _munge_sys_path(dir) \ : try: run_path(str(path), run_name='__main__') except Exception as err: print_exc(file=sys.stderr) p.returncode = 1 except SystemExit as err: # We need to handle SystemExit as if we were the python # interpreter. From the python documentation: # # If [err.code] is an integer, it specifies the system exit # status (passed to C's exit() function); if it is None, the # exit status is zero; if it has another type (such as a # string), the object's value is printed and the exit status is # one. # # https://docs.python.org/3/library/exceptions.html?highlight=systemexit#SystemExit if isinstance(err.code, int): p.returncode = err.code elif err.code is None: p.returncode = 0 else: print(err.code, file=sys.stderr) p.returncode = 1 p.stdout = stdout.getvalue() assert isinstance(p.returncode, int), repr(p.returncode) return p