def _spawn_and_communicate(
client: pyspawner.Client,
indented_code: str,
stdin: bytes = b"",
chroot_dir: Optional[Path] = None,
network_config: Optional[pyspawner.NetworkConfig] = None,
skip_sandbox_except: FrozenSet[str] = frozenset(),
) -> Tuple[int, bytes, bytes]:
"""Spawn, execute `indented_code`, and return (exitcode, stdout, stderr).
This will never error.
"""
with _spawned_child_context(
client,
args=[indented_code],
sandbox_config=pyspawner.SandboxConfig(
chroot_dir=chroot_dir,
network=network_config,
skip_sandbox_except=skip_sandbox_except,
),
) as subprocess:
subprocess.stdin.write(stdin)
subprocess.stdin.close()
stdout = subprocess.stdout.read()
stderr = subprocess.stderr.read()
subprocess.stdout.close()
subprocess.stderr.close()
_, status = subprocess.wait(0)
if os.WIFSIGNALED(status):
exitcode = -os.WTERMSIG(status)
elif os.WIFEXITED(status):
exitcode = os.WEXITSTATUS(status)
else:
raise OSError("Unexpected status: %d" % status)
return exitcode, stdout, stderr
def _spawned_child_context(
client: pyspawner.Client,
args: List[Any] = [],
sandbox_config: pyspawner.SandboxConfig = pyspawner.SandboxConfig(),
) -> ContextManager[pyspawner.ChildProcess]:
subprocess = client.spawn_child(args,
process_name="pyspawner-test",
sandbox_config=sandbox_config)
try:
yield subprocess
finally:
try:
subprocess.stdout.read()
except ValueError:
pass # stdout already closed
try:
subprocess.stderr.read()
except ValueError:
pass # stderr already closed
try:
subprocess.kill()
except ProcessLookupError:
pass
try:
subprocess.wait(0)
except ChildProcessError:
pass
def _run_in_child(
self,
*,
chroot_dir: Path,
network_config: Optional[pyspawner.NetworkConfig],
compiled_module: CompiledModule,
timeout: float,
result: Any,
function: str,
args: List[Any],
) -> None:
"""
Fork a child process to run `function` with `args`.
`args` must be Thrift data types. `result` must also be a Thrift type --
its `.read()` function will be called, which may produce an error if
the child process has a bug. (EOFError is very likely.)
Raise ModuleExitedError if the child process did not behave as expected.
Raise ModuleTimeoutError if it did not exit after a delay -- or if it
closed its file descriptors long before it exited.
"""
limit_time = time.time() + timeout
module_process = self._pyspawner.spawn_child(
args=[compiled_module, function, args],
process_name=compiled_module.module_slug,
sandbox_config=pyspawner.SandboxConfig(
chroot_dir=chroot_dir, network=network_config
),
)
# stdout is Thrift package; stderr is logs
output_reader = ChildReader(
module_process.stdout.fileno(), OUTPUT_BUFFER_MAX_BYTES
)
log_reader = ChildReader(module_process.stderr.fileno(), LOG_BUFFER_MAX_BYTES)
# Read until the child closes its stdout and stderr
with selectors.DefaultSelector() as selector:
selector.register(output_reader.fileno, selectors.EVENT_READ)
selector.register(log_reader.fileno, selectors.EVENT_READ)
timed_out = False
while selector.get_map():
remaining = limit_time - time.time()
if remaining <= 0:
if not timed_out:
timed_out = True
module_process.kill() # untrusted code could ignore SIGTERM
timeout = None # wait as long as it takes for everything to die
# Fall through. After SIGKILL the child will close each fd,
# sending EOF to us. That means the selector _must_ return.
else:
timeout = remaining # wait until we reach our timeout
events = selector.select(timeout=timeout)
ready = frozenset(key.fd for key, _ in events)
for reader in (output_reader, log_reader):
if reader.fileno in ready:
reader.ingest()
if reader.eof:
selector.unregister(reader.fileno)
# The child closed its fds, so it should die soon. If it doesn't, that's
# a bug -- so kill -9 it!
#
# os.wait() has no timeout option, and asyncio messes with signals so
# we won't use those. Spin until the process dies, and force-kill if we
# spin too long.
for _ in range(DEAD_PROCESS_N_WAITS):
pid, exit_status = module_process.wait(os.WNOHANG)
if pid != 0: # pid==0 means process is still running
break
time.sleep(DEAD_PROCESS_WAIT_POLL_INTERVAL)
else:
# we waited and waited. No luck. Dead module. Kill it.
timed_out = True
module_process.kill()
_, exit_status = module_process.wait(0)
if os.WIFEXITED(exit_status):
exit_code = os.WEXITSTATUS(exit_status)
elif os.WIFSIGNALED(exit_status):
exit_code = -os.WTERMSIG(exit_status)
else:
raise RuntimeError("Unhandled wait() status: %r" % exit_status)
if timed_out:
raise ModuleTimeoutError
if exit_code != 0:
raise ModuleExitedError(exit_code, log_reader.to_str())
transport = thrift.transport.TTransport.TMemoryBuffer(output_reader.buffer)
protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport)
try:
result.read(protocol)
except EOFError: # TODO handle other errors Thrift may throw
raise ModuleExitedError(exit_code, log_reader.to_str()) from None
# We should be at the end of the output now. If we aren't, that means
# the child wrote too much.
if transport.read(1) != b"":
raise ModuleExitedError(exit_code, log_reader.to_str())
if log_reader.buffer:
logger.info("Output from module process: %s", log_reader.to_str())
return result