def test_eintr_retry_call(self): record_calls = [] def fake_os_func(*args): record_calls.append(args) if len(record_calls) == 2: raise OSError(errno.EINTR, "fake interrupted system call") return tuple(reversed(args)) self.assertEqual((999, 256), subprocess._eintr_retry_call(fake_os_func, 256, 999)) self.assertEqual([(256, 999)], record_calls) # This time there will be an EINTR so it will loop once. self.assertEqual((666,), subprocess._eintr_retry_call(fake_os_func, 666)) self.assertEqual([(256, 999), (666,), (666,)], record_calls)
def _test_eintr_retry_call(self): record_calls = [] def fake_os_func(*args): record_calls.append(args) if len(record_calls) == 2: raise OSError(errno.EINTR, "fake interrupted system call") return tuple(reversed(args)) self.assertEqual((999, 256), subprocess._eintr_retry_call(fake_os_func, 256, 999)) self.assertEqual([(256, 999)], record_calls) # This time there will be an EINTR so it will loop once. self.assertEqual((666,), subprocess._eintr_retry_call(fake_os_func, 666)) self.assertEqual([(256, 999), (666,), (666,)], record_calls)
def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" start = time.time() while self.returncode is None and (timeout is None or time.time() - start < timeout): try: pid, sts = subprocess._eintr_retry_call(os.waitpid, self.pid, os.WNOHANG) # WNOHANG in action! if pid == 0: time.sleep(0.1) continue except OSError as e: if e.errno != errno.ECHILD: raise # This happens if SIGCLD is set to be ignored or waiting # for child processes has otherwise been disabled for our # process. This child is dead, we can't get the status. pid = self.pid sts = 0 # Check the pid and loop as waitpid has been known to return # 0 even without WNOHANG in odd situations. issue14396. if pid == self.pid: self._handle_exitstatus(sts) return self.returncode
def _wait(): finished = None interrupted = False try: if hasattr(os, "wait4"): try: if hasattr(subprocess, "_eintr_retry_call"): pid, sts, rusage = subprocess._eintr_retry_call( os.wait4, self._process.pid, 0) else: # PEP 475 pid, sts, rusage = os.wait4(self._process.pid, 0) finished = time.time() self._process._handle_exitstatus(sts) for field in [ "ru_idrss", "ru_inblock", "ru_isrss", "ru_ixrss", "ru_majflt", "ru_maxrss", "ru_minflt", "ru_msgrcv", "ru_msgsnd", "ru_nivcsw", "ru_nsignals", "ru_nswap", "ru_nvcsw", "ru_oublock", "ru_stime", "ru_utime", ]: if hasattr(rusage, field): self._metrics[field.replace("ru_", "")] = getattr( rusage, field) except OSError as exc: if exc.errno == errno.ECHILD: yatest_logger.debug( "Process resource usage is not available as process finished before wait4 was called" ) else: raise except SignalInterruptionError: interrupted = True raise finally: if not interrupted: self._process.wait( ) # this has to be here unconditionally, so that all process properties are set if not finished: finished = time.time() self._metrics["wtime"] = round(finished - self._started, 3)
def wait(self, timeout=None, poll_initial_interval=0.001, poll_max_interval=0.05): # pylint: disable=arguments-differ """Implements python3's timeout support. Raises: - TimeoutExpired when more than timeout seconds were spent waiting for the process. """ assert timeout is None or isinstance(timeout, (int, float)), timeout if timeout is None: super(Popen, self).wait() elif six.PY3: super(Popen, self).wait(timeout) elif self.returncode is None: if sys.platform == 'win32': WAIT_TIMEOUT = 258 result = subprocess._subprocess.WaitForSingleObject( self._handle, int(timeout * 1000)) if result == WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = subprocess._subprocess.GetExitCodeProcess( self._handle) else: # If you think the following code is horrible, it's because it is # inspired by python3's stdlib. end = time.time() + timeout delay = poll_initial_interval while True: try: pid, sts = subprocess._eintr_retry_call( os.waitpid, self.pid, os.WNOHANG) except OSError as e: if e.errno != errno.ECHILD: raise pid = self.pid sts = 0 if pid == self.pid: # This sets self.returncode. self._handle_exitstatus(sts) break remaining = end - time.time() if remaining <= 0: raise TimeoutExpired(self.args, timeout) delay = min(delay * 2, remaining, poll_max_interval) time.sleep(delay) if not self.end: # communicate() uses wait() internally. self.end = time.time() self._cleanup() return self.returncode
def wait(self, timeout=None): # pylint: disable=arguments-differ """Implements python3's timeout support. Raises: - TimeoutExpired when more than timeout seconds were spent waiting for the process. """ assert timeout is None or isinstance(timeout, (int, float)), timeout if timeout is None: super(Popen, self).wait() elif self.returncode is None: if subprocess.mswindows: WAIT_TIMEOUT = 258 result = subprocess._subprocess.WaitForSingleObject( self._handle, int(timeout * 1000)) if result == WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = subprocess._subprocess.GetExitCodeProcess( self._handle) else: # If you think the following code is horrible, it's because it is # inspired by python3's stdlib. end = time.time() + timeout delay = 0.001 while True: try: pid, sts = subprocess._eintr_retry_call( os.waitpid, self.pid, os.WNOHANG) except OSError as e: if e.errno != errno.ECHILD: raise pid = self.pid sts = 0 if pid == self.pid: # This sets self.returncode. self._handle_exitstatus(sts) break remaining = end - time.time() if remaining <= 0: raise TimeoutExpired(self.args, timeout) delay = min(delay * 2, remaining, .05) time.sleep(delay) if not self.end: # communicate() uses wait() internally. self.end = time.time() return self.returncode
def _wait_nohang(self): pid = None try: pid, sts = _eintr_retry_call(os.waitpid, self.pid, os.WNOHANG) except OSError as exc: if exc.errno != errno.ECHILD: raise # This happens if SIGCLD is set to be ignored or waiting # for child processes has otherwise been disabled for our # process. This child is dead, we can't get the status. pid = self.pid sts = 0 # Check the pid and loop as waitpid has been known to return # 0 even without WNOHANG in odd situations. issue14396. if pid == self.pid: self._handle_exitstatus(sts) # pylint: disable=no-member return self.returncode
def wait(self, timeout=None): # pylint: disable=arguments-differ """Implements python3's timeout support.""" if timeout is None: ret = super(Popen, self).wait() else: if subprocess.mswindows: WAIT_TIMEOUT = 258 if self.returncode is None: result = subprocess._subprocess.WaitForSingleObject( self._handle, int(timeout * 1000)) if result == WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = subprocess._subprocess.GetExitCodeProcess( self._handle) else: # If you think the following code is horrible, it's because it is # inspired by python3's stdlib. end = time.time() + timeout delay = 0.001 while True: try: pid, sts = subprocess._eintr_retry_call( os.waitpid, self.pid, os.WNOHANG) except OSError as e: if e.errno != errno.ECHILD: raise pid = self.pid sts = 0 if pid == self.pid: self._handle_exitstatus(sts) break remaining = end - time.time() if remaining <= 0: raise TimeoutExpired(self.args, timeout) delay = min(delay * 2, remaining, .05) time.sleep(delay) ret = self.returncode if not self.end: # communicate() uses wait() internally. self.end = time.time() return ret
def _wait(): finished = None try: if hasattr(os, "wait4"): try: pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0) finished = time.time() self._process._handle_exitstatus(sts) for field in [ "ru_idrss", "ru_inblock", "ru_isrss", "ru_ixrss", "ru_majflt", "ru_maxrss", "ru_minflt", "ru_msgrcv", "ru_msgsnd", "ru_nivcsw", "ru_nsignals", "ru_nswap", "ru_nvcsw", "ru_oublock", "ru_stime", "ru_utime", ]: if hasattr(rusage, field): self._metrics[field.replace("ru_", "")] = getattr(rusage, field) except OSError as exc: if exc.errno == errno.ECHILD: yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") else: raise finally: self._process.wait() # this has to be here unconditionally, so that all process properties are set if not finished: finished = time.time() self._metrics["wtime"] = round(finished - self._started, 3)
def _execute_child(self, py_func, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = os.pipe() try: try: self._set_cloexec_flag(errpipe_write) gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = os.fork() except: if gc_was_enabled: gc.enable() raise self._child_created = True if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # Dup fds for child if p2cread is not None: os.dup2(p2cread, 0) if c2pwrite is not None: os.dup2(c2pwrite, 1) if errwrite is not None: os.dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the same # fd more than once, or standard fds. if p2cread is not None and p2cread not in (0,): os.close(p2cread) if c2pwrite is not None and c2pwrite not in (p2cread, 1): os.close(c2pwrite) if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): os.close(errwrite) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() child_stdin = os.fdopen(0, "r") child_stdout = os.fdopen(1, "w") child_stderr = os.fdopen(2, "w") #call the child function py_func(child_stdin, child_stdout, child_stderr) child_stdin.close() child_stdout.close() child_stderr.close() except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) # This exitcode won't be reported to applications, so it # really doesn't matter what we return. #os._exit(0) # in the case of extproc, the exit status does # matter, we want the exit status to be 0 os._exit(0) # Parent if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: # be sure the FD is closed no matter what #pass os.close(errpipe_read) if data != "": print data _eintr_retry_call(os.waitpid, self.pid, 0) child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception
def _execute_child(*args): (self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = _unpack_args(args) if preexec_fn: return wrapped_execute_child( self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) for fd in (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): if fd is not None: _set_cloexec_flag(fd, False) errpipe_read, errpipe_write = os.pipe() data = base64.encodestring( pickle.dumps( dict( args=args, executable=executable, close_fds=close_fds, cwd=cwd, env=env, shell=shell, p2cread=p2cread, p2cwrite=p2cwrite, c2pread=c2pread, c2pwrite=c2pwrite, errread=errread, errwrite=errwrite, errpipe_read=errpipe_read, errpipe_write=errpipe_write, ), pickle.HIGHEST_PROTOCOL)) use_file = len(data) >= ARG_MAX if use_file: with tempfile.NamedTemporaryFile( delete=False) as the_file: the_file.write(data) argv = (ctypes.c_char_p * 5)(sys.executable, __file__, '@', the_file.name, 0) else: argv = (ctypes.c_char_p * 4)(sys.executable, __file__, data, 0) pid = ctypes.c_int() os_environ = (ctypes.c_char_p * (len(os.environ) + 1))(*([ '{0}={1}'.format(*value) for value in os.environ.iteritems() ] + [0])) try: ret = posix_spawn( ctypes.byref(pid), ctypes.c_char_p(sys.executable), ctypes.c_void_p(), ctypes.c_void_p(), ctypes.cast(argv, ctypes.POINTER(ctypes.c_char_p)), ctypes.cast(os_environ, ctypes.POINTER(ctypes.c_char_p))) err = ctypes.get_errno() os.close(errpipe_write) if ret: raise OSError(err, os.strerror(err)) self.pid = pid.value self._child_created = True # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) if data: try: _eintr_retry_call(os.waitpid, self.pid, 0) except OSError as e: if e.errno != errno.ECHILD: raise raise pickle.loads(data) finally: if use_file: try: os.unlink(the_file.name) except: pass os.close(errpipe_read) if p2cread is not None and p2cwrite is not None: _close_in_parent(p2cread, to_close) if c2pwrite is not None and c2pread is not None: _close_in_parent(c2pwrite, to_close) if errwrite is not None and errread is not None: _close_in_parent(errwrite, to_close)