def _spawn_and_communicate( client: pyspawner.Client, indented_code: str, stdin: bytes = b"", chroot_dir: Optional[Path] = None, network_config: Optional[pyspawner.NetworkConfig] = None, skip_sandbox_except: FrozenSet[str] = frozenset(), ) -> Tuple[int, bytes, bytes]: """Spawn, execute `indented_code`, and return (exitcode, stdout, stderr). This will never error. """ with _spawned_child_context( client, args=[indented_code], sandbox_config=pyspawner.SandboxConfig( chroot_dir=chroot_dir, network=network_config, skip_sandbox_except=skip_sandbox_except, ), ) as subprocess: subprocess.stdin.write(stdin) subprocess.stdin.close() stdout = subprocess.stdout.read() stderr = subprocess.stderr.read() subprocess.stdout.close() subprocess.stderr.close() _, status = subprocess.wait(0) if os.WIFSIGNALED(status): exitcode = -os.WTERMSIG(status) elif os.WIFEXITED(status): exitcode = os.WEXITSTATUS(status) else: raise OSError("Unexpected status: %d" % status) return exitcode, stdout, stderr
def _spawned_child_context( client: pyspawner.Client, args: List[Any] = [], sandbox_config: pyspawner.SandboxConfig = pyspawner.SandboxConfig(), ) -> ContextManager[pyspawner.ChildProcess]: subprocess = client.spawn_child(args, process_name="pyspawner-test", sandbox_config=sandbox_config) try: yield subprocess finally: try: subprocess.stdout.read() except ValueError: pass # stdout already closed try: subprocess.stderr.read() except ValueError: pass # stderr already closed try: subprocess.kill() except ProcessLookupError: pass try: subprocess.wait(0) except ChildProcessError: pass
def _run_in_child( self, *, chroot_dir: Path, network_config: Optional[pyspawner.NetworkConfig], compiled_module: CompiledModule, timeout: float, result: Any, function: str, args: List[Any], ) -> None: """ Fork a child process to run `function` with `args`. `args` must be Thrift data types. `result` must also be a Thrift type -- its `.read()` function will be called, which may produce an error if the child process has a bug. (EOFError is very likely.) Raise ModuleExitedError if the child process did not behave as expected. Raise ModuleTimeoutError if it did not exit after a delay -- or if it closed its file descriptors long before it exited. """ limit_time = time.time() + timeout module_process = self._pyspawner.spawn_child( args=[compiled_module, function, args], process_name=compiled_module.module_slug, sandbox_config=pyspawner.SandboxConfig( chroot_dir=chroot_dir, network=network_config ), ) # stdout is Thrift package; stderr is logs output_reader = ChildReader( module_process.stdout.fileno(), OUTPUT_BUFFER_MAX_BYTES ) log_reader = ChildReader(module_process.stderr.fileno(), LOG_BUFFER_MAX_BYTES) # Read until the child closes its stdout and stderr with selectors.DefaultSelector() as selector: selector.register(output_reader.fileno, selectors.EVENT_READ) selector.register(log_reader.fileno, selectors.EVENT_READ) timed_out = False while selector.get_map(): remaining = limit_time - time.time() if remaining <= 0: if not timed_out: timed_out = True module_process.kill() # untrusted code could ignore SIGTERM timeout = None # wait as long as it takes for everything to die # Fall through. After SIGKILL the child will close each fd, # sending EOF to us. That means the selector _must_ return. else: timeout = remaining # wait until we reach our timeout events = selector.select(timeout=timeout) ready = frozenset(key.fd for key, _ in events) for reader in (output_reader, log_reader): if reader.fileno in ready: reader.ingest() if reader.eof: selector.unregister(reader.fileno) # The child closed its fds, so it should die soon. If it doesn't, that's # a bug -- so kill -9 it! # # os.wait() has no timeout option, and asyncio messes with signals so # we won't use those. Spin until the process dies, and force-kill if we # spin too long. for _ in range(DEAD_PROCESS_N_WAITS): pid, exit_status = module_process.wait(os.WNOHANG) if pid != 0: # pid==0 means process is still running break time.sleep(DEAD_PROCESS_WAIT_POLL_INTERVAL) else: # we waited and waited. No luck. Dead module. Kill it. timed_out = True module_process.kill() _, exit_status = module_process.wait(0) if os.WIFEXITED(exit_status): exit_code = os.WEXITSTATUS(exit_status) elif os.WIFSIGNALED(exit_status): exit_code = -os.WTERMSIG(exit_status) else: raise RuntimeError("Unhandled wait() status: %r" % exit_status) if timed_out: raise ModuleTimeoutError if exit_code != 0: raise ModuleExitedError(exit_code, log_reader.to_str()) transport = thrift.transport.TTransport.TMemoryBuffer(output_reader.buffer) protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport) try: result.read(protocol) except EOFError: # TODO handle other errors Thrift may throw raise ModuleExitedError(exit_code, log_reader.to_str()) from None # We should be at the end of the output now. If we aren't, that means # the child wrote too much. if transport.read(1) != b"": raise ModuleExitedError(exit_code, log_reader.to_str()) if log_reader.buffer: logger.info("Output from module process: %s", log_reader.to_str()) return result