def _fast_run_command_target(command: List[str],
w: Connection,
stdin: Optional[int] = None,
stdout: Optional[int] = None,
stderr: Optional[int] = None) -> None:
"""Low-level command invoker with low overhead.
Writes the CommandResult instance to the provided connection w.
Uses pure os wrappers around libc commands, so it's pretty fast.
"""
# this function is to be run in a dedicated process anyway, so we can overwrite
# current process' fds for more accurate child execution time measurement
if stdin is not None:
os.dup2(stdin, 0, inheritable=True)
if stdout is not None:
os.dup2(stdout, 1, inheritable=True)
if stderr is not None:
os.dup2(stderr, 2, inheritable=True)
_close_all_fds(STANDARD_FDS | {w.fileno()})
pid = os.fork()
if pid == 0: # in child
os.close(w.fileno())
os.execvp(command[0], command)
os._exit(1) # this should never happen (unless execvp fails)
opts = os.WUNTRACED
start_time = time.monotonic()
_, status = os.waitpid(pid, opts)
end_time = time.monotonic()
rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
real_time = end_time - start_time
cpu_time = rusage.ru_utime + rusage.ru_stime
max_memory = rusage.ru_maxrss
# mac os shows result in bytes, normal OSes -- in kilobytes
if platform.platform() != 'darwin':
max_memory *= 1024
if os.WIFSIGNALED(status):
code = 128 + os.WTERMSIG(status)
else:
code = os.WEXITSTATUS(status)
result = CommandStats(
return_code=code,
real_time=math.ceil(real_time * 1000),
cpu_time=math.ceil(cpu_time * 1000),
max_memory=math.ceil(max_memory),
)
w.send(result)
def _safe_queue_put(self, worker_id: int, item: Any,
queue: mp.JoinableQueue, rx: Connection) -> bool:
while True:
# First we have to check to make sure the parent process is still alive
# and consuming from the queue because there are circumstances where the
# parent process can or exit stop consuming without automatically cleaning up
# its children (the workers).
# For example, when the parent process is killed with `kill -9`.
# So the first thing we do is check to see if the parent has notified
# us (the worker) to stop through the rx (receiver) connection.
# Of course this only works if the parent was able to send out a notification,
# which may not always be the case. So we have a backup check below.
if rx.poll():
logger.warning(
"worker %d received stop message from parent, exiting now",
worker_id)
queue.cancel_join_thread()
return False
# The is the backup check.
# The file descriptor associated with the rx (receiver) connection will
# be readable if and only if the parent process has exited.
# NOTE (epwalsh): this doesn't work on Mac OS X with `start_method == "fork"`
# for some reason, i.e. the file descriptor doesn't show as readable
# after the parent process has died.
fds, _, _ = select.select([rx.fileno()], [], [], 0)
if fds:
logger.warning(
"worker %d parent process has died, exiting now",
worker_id)
queue.cancel_join_thread()
return False
# If we're down here the parent process is still alive to the best of our
# knowledge, so we can continue putting things on the queue.
try:
queue.put(item, True, 0.1)
return True
except Full:
continue
