def _fast_run_command_target(command: List[str], w: Connection, stdin: Optional[int] = None, stdout: Optional[int] = None, stderr: Optional[int] = None) -> None: """Low-level command invoker with low overhead. Writes the CommandResult instance to the provided connection w. Uses pure os wrappers around libc commands, so it's pretty fast. """ # this function is to be run in a dedicated process anyway, so we can overwrite # current process' fds for more accurate child execution time measurement if stdin is not None: os.dup2(stdin, 0, inheritable=True) if stdout is not None: os.dup2(stdout, 1, inheritable=True) if stderr is not None: os.dup2(stderr, 2, inheritable=True) _close_all_fds(STANDARD_FDS | {w.fileno()}) pid = os.fork() if pid == 0: # in child os.close(w.fileno()) os.execvp(command[0], command) os._exit(1) # this should never happen (unless execvp fails) opts = os.WUNTRACED start_time = time.monotonic() _, status = os.waitpid(pid, opts) end_time = time.monotonic() rusage = resource.getrusage(resource.RUSAGE_CHILDREN) real_time = end_time - start_time cpu_time = rusage.ru_utime + rusage.ru_stime max_memory = rusage.ru_maxrss # mac os shows result in bytes, normal OSes -- in kilobytes if platform.platform() != 'darwin': max_memory *= 1024 if os.WIFSIGNALED(status): code = 128 + os.WTERMSIG(status) else: code = os.WEXITSTATUS(status) result = CommandStats( return_code=code, real_time=math.ceil(real_time * 1000), cpu_time=math.ceil(cpu_time * 1000), max_memory=math.ceil(max_memory), ) w.send(result)
def _safe_queue_put(self, worker_id: int, item: Any, queue: mp.JoinableQueue, rx: Connection) -> bool: while True: # First we have to check to make sure the parent process is still alive # and consuming from the queue because there are circumstances where the # parent process can or exit stop consuming without automatically cleaning up # its children (the workers). # For example, when the parent process is killed with `kill -9`. # So the first thing we do is check to see if the parent has notified # us (the worker) to stop through the rx (receiver) connection. # Of course this only works if the parent was able to send out a notification, # which may not always be the case. So we have a backup check below. if rx.poll(): logger.warning( "worker %d received stop message from parent, exiting now", worker_id) queue.cancel_join_thread() return False # The is the backup check. # The file descriptor associated with the rx (receiver) connection will # be readable if and only if the parent process has exited. # NOTE (epwalsh): this doesn't work on Mac OS X with `start_method == "fork"` # for some reason, i.e. the file descriptor doesn't show as readable # after the parent process has died. fds, _, _ = select.select([rx.fileno()], [], [], 0) if fds: logger.warning( "worker %d parent process has died, exiting now", worker_id) queue.cancel_join_thread() return False # If we're down here the parent process is still alive to the best of our # knowledge, so we can continue putting things on the queue. try: queue.put(item, True, 0.1) return True except Full: continue