def test_eintr_retry_call(self):
record_calls = []
def fake_os_func(*args):
record_calls.append(args)
if len(record_calls) == 2:
raise OSError(errno.EINTR, "fake interrupted system call")
return tuple(reversed(args))
self.assertEqual((999, 256), subprocess._eintr_retry_call(fake_os_func, 256, 999))
self.assertEqual([(256, 999)], record_calls)
# This time there will be an EINTR so it will loop once.
self.assertEqual((666,), subprocess._eintr_retry_call(fake_os_func, 666))
self.assertEqual([(256, 999), (666,), (666,)], record_calls)
def _test_eintr_retry_call(self):
record_calls = []
def fake_os_func(*args):
record_calls.append(args)
if len(record_calls) == 2:
raise OSError(errno.EINTR, "fake interrupted system call")
return tuple(reversed(args))
self.assertEqual((999, 256),
subprocess._eintr_retry_call(fake_os_func, 256, 999))
self.assertEqual([(256, 999)], record_calls)
# This time there will be an EINTR so it will loop once.
self.assertEqual((666,),
subprocess._eintr_retry_call(fake_os_func, 666))
self.assertEqual([(256, 999), (666,), (666,)], record_calls)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
start = time.time()
while self.returncode is None and (timeout is None or time.time() - start < timeout):
try:
pid, sts = subprocess._eintr_retry_call(os.waitpid, self.pid, os.WNOHANG)
# WNOHANG in action!
if pid == 0:
time.sleep(0.1)
continue
except OSError as e:
if e.errno != errno.ECHILD:
raise
# This happens if SIGCLD is set to be ignored or waiting
# for child processes has otherwise been disabled for our
# process. This child is dead, we can't get the status.
pid = self.pid
sts = 0
# Check the pid and loop as waitpid has been known to return
# 0 even without WNOHANG in odd situations. issue14396.
if pid == self.pid:
self._handle_exitstatus(sts)
return self.returncode
def _wait():
finished = None
interrupted = False
try:
if hasattr(os, "wait4"):
try:
if hasattr(subprocess, "_eintr_retry_call"):
pid, sts, rusage = subprocess._eintr_retry_call(
os.wait4, self._process.pid, 0)
else:
# PEP 475
pid, sts, rusage = os.wait4(self._process.pid, 0)
finished = time.time()
self._process._handle_exitstatus(sts)
for field in [
"ru_idrss",
"ru_inblock",
"ru_isrss",
"ru_ixrss",
"ru_majflt",
"ru_maxrss",
"ru_minflt",
"ru_msgrcv",
"ru_msgsnd",
"ru_nivcsw",
"ru_nsignals",
"ru_nswap",
"ru_nvcsw",
"ru_oublock",
"ru_stime",
"ru_utime",
]:
if hasattr(rusage, field):
self._metrics[field.replace("ru_",
"")] = getattr(
rusage, field)
except OSError as exc:
if exc.errno == errno.ECHILD:
yatest_logger.debug(
"Process resource usage is not available as process finished before wait4 was called"
)
else:
raise
except SignalInterruptionError:
interrupted = True
raise
finally:
if not interrupted:
self._process.wait(
) # this has to be here unconditionally, so that all process properties are set
if not finished:
finished = time.time()
self._metrics["wtime"] = round(finished - self._started, 3)
def wait(self,
timeout=None,
poll_initial_interval=0.001,
poll_max_interval=0.05): # pylint: disable=arguments-differ
"""Implements python3's timeout support.
Raises:
- TimeoutExpired when more than timeout seconds were spent waiting for the
process.
"""
assert timeout is None or isinstance(timeout, (int, float)), timeout
if timeout is None:
super(Popen, self).wait()
elif six.PY3:
super(Popen, self).wait(timeout)
elif self.returncode is None:
if sys.platform == 'win32':
WAIT_TIMEOUT = 258
result = subprocess._subprocess.WaitForSingleObject(
self._handle, int(timeout * 1000))
if result == WAIT_TIMEOUT:
raise TimeoutExpired(self.args, timeout)
self.returncode = subprocess._subprocess.GetExitCodeProcess(
self._handle)
else:
# If you think the following code is horrible, it's because it is
# inspired by python3's stdlib.
end = time.time() + timeout
delay = poll_initial_interval
while True:
try:
pid, sts = subprocess._eintr_retry_call(
os.waitpid, self.pid, os.WNOHANG)
except OSError as e:
if e.errno != errno.ECHILD:
raise
pid = self.pid
sts = 0
if pid == self.pid:
# This sets self.returncode.
self._handle_exitstatus(sts)
break
remaining = end - time.time()
if remaining <= 0:
raise TimeoutExpired(self.args, timeout)
delay = min(delay * 2, remaining, poll_max_interval)
time.sleep(delay)
if not self.end:
# communicate() uses wait() internally.
self.end = time.time()
self._cleanup()
return self.returncode
def wait(self, timeout=None): # pylint: disable=arguments-differ
"""Implements python3's timeout support.
Raises:
- TimeoutExpired when more than timeout seconds were spent waiting for the
process.
"""
assert timeout is None or isinstance(timeout, (int, float)), timeout
if timeout is None:
super(Popen, self).wait()
elif self.returncode is None:
if subprocess.mswindows:
WAIT_TIMEOUT = 258
result = subprocess._subprocess.WaitForSingleObject(
self._handle, int(timeout * 1000))
if result == WAIT_TIMEOUT:
raise TimeoutExpired(self.args, timeout)
self.returncode = subprocess._subprocess.GetExitCodeProcess(
self._handle)
else:
# If you think the following code is horrible, it's because it is
# inspired by python3's stdlib.
end = time.time() + timeout
delay = 0.001
while True:
try:
pid, sts = subprocess._eintr_retry_call(
os.waitpid, self.pid, os.WNOHANG)
except OSError as e:
if e.errno != errno.ECHILD:
raise
pid = self.pid
sts = 0
if pid == self.pid:
# This sets self.returncode.
self._handle_exitstatus(sts)
break
remaining = end - time.time()
if remaining <= 0:
raise TimeoutExpired(self.args, timeout)
delay = min(delay * 2, remaining, .05)
time.sleep(delay)
if not self.end:
# communicate() uses wait() internally.
self.end = time.time()
return self.returncode
def _wait_nohang(self):
pid = None
try:
pid, sts = _eintr_retry_call(os.waitpid, self.pid, os.WNOHANG)
except OSError as exc:
if exc.errno != errno.ECHILD:
raise
# This happens if SIGCLD is set to be ignored or waiting
# for child processes has otherwise been disabled for our
# process. This child is dead, we can't get the status.
pid = self.pid
sts = 0
# Check the pid and loop as waitpid has been known to return
# 0 even without WNOHANG in odd situations. issue14396.
if pid == self.pid:
self._handle_exitstatus(sts) # pylint: disable=no-member
return self.returncode
def wait(self, timeout=None): # pylint: disable=arguments-differ
"""Implements python3's timeout support."""
if timeout is None:
ret = super(Popen, self).wait()
else:
if subprocess.mswindows:
WAIT_TIMEOUT = 258
if self.returncode is None:
result = subprocess._subprocess.WaitForSingleObject(
self._handle, int(timeout * 1000))
if result == WAIT_TIMEOUT:
raise TimeoutExpired(self.args, timeout)
self.returncode = subprocess._subprocess.GetExitCodeProcess(
self._handle)
else:
# If you think the following code is horrible, it's because it is
# inspired by python3's stdlib.
end = time.time() + timeout
delay = 0.001
while True:
try:
pid, sts = subprocess._eintr_retry_call(
os.waitpid, self.pid, os.WNOHANG)
except OSError as e:
if e.errno != errno.ECHILD:
raise
pid = self.pid
sts = 0
if pid == self.pid:
self._handle_exitstatus(sts)
break
remaining = end - time.time()
if remaining <= 0:
raise TimeoutExpired(self.args, timeout)
delay = min(delay * 2, remaining, .05)
time.sleep(delay)
ret = self.returncode
if not self.end:
# communicate() uses wait() internally.
self.end = time.time()
return ret
def _wait():
finished = None
try:
if hasattr(os, "wait4"):
try:
pid, sts, rusage = subprocess._eintr_retry_call(os.wait4, self._process.pid, 0)
finished = time.time()
self._process._handle_exitstatus(sts)
for field in [
"ru_idrss",
"ru_inblock",
"ru_isrss",
"ru_ixrss",
"ru_majflt",
"ru_maxrss",
"ru_minflt",
"ru_msgrcv",
"ru_msgsnd",
"ru_nivcsw",
"ru_nsignals",
"ru_nswap",
"ru_nvcsw",
"ru_oublock",
"ru_stime",
"ru_utime",
]:
if hasattr(rusage, field):
self._metrics[field.replace("ru_", "")] = getattr(rusage, field)
except OSError as exc:
if exc.errno == errno.ECHILD:
yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called")
else:
raise
finally:
self._process.wait() # this has to be here unconditionally, so that all process properties are set
if not finished:
finished = time.time()
self._metrics["wtime"] = round(finished - self._started, 3)
def _execute_child(self, py_func, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (POSIX version)"""
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = os.pipe()
try:
try:
self._set_cloexec_flag(errpipe_write)
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = os.fork()
except:
if gc_was_enabled:
gc.enable()
raise
self._child_created = True
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# Dup fds for child
if p2cread is not None:
os.dup2(p2cread, 0)
if c2pwrite is not None:
os.dup2(c2pwrite, 1)
if errwrite is not None:
os.dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the same
# fd more than once, or standard fds.
if p2cread is not None and p2cread not in (0,):
os.close(p2cread)
if c2pwrite is not None and c2pwrite not in (p2cread, 1):
os.close(c2pwrite)
if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
os.close(errwrite)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
child_stdin = os.fdopen(0, "r")
child_stdout = os.fdopen(1, "w")
child_stderr = os.fdopen(2, "w")
#call the child function
py_func(child_stdin, child_stdout, child_stderr)
child_stdin.close()
child_stdout.close()
child_stderr.close()
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
# This exitcode won't be reported to applications, so it
# really doesn't matter what we return.
#os._exit(0)
# in the case of extproc, the exit status does
# matter, we want the exit status to be 0
os._exit(0)
# Parent
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
os.close(errwrite)
# Wait for exec to fail or succeed; possibly raising exception
# Exception limited to 1M
data = _eintr_retry_call(os.read, errpipe_read, 1048576)
finally:
# be sure the FD is closed no matter what
#pass
os.close(errpipe_read)
if data != "":
print data
_eintr_retry_call(os.waitpid, self.pid, 0)
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _execute_child(*args):
(self, args, executable, preexec_fn, close_fds, cwd, env,
universal_newlines, startupinfo, creationflags, shell,
to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread,
errwrite) = _unpack_args(args)
if preexec_fn:
return wrapped_execute_child(
self, args, executable, preexec_fn, close_fds, cwd,
env, universal_newlines, startupinfo,
creationflags, shell, p2cread, p2cwrite, c2pread,
c2pwrite, errread, errwrite)
for fd in (p2cread, p2cwrite, c2pread, c2pwrite, errread,
errwrite):
if fd is not None:
_set_cloexec_flag(fd, False)
errpipe_read, errpipe_write = os.pipe()
data = base64.encodestring(
pickle.dumps(
dict(
args=args,
executable=executable,
close_fds=close_fds,
cwd=cwd,
env=env,
shell=shell,
p2cread=p2cread,
p2cwrite=p2cwrite,
c2pread=c2pread,
c2pwrite=c2pwrite,
errread=errread,
errwrite=errwrite,
errpipe_read=errpipe_read,
errpipe_write=errpipe_write,
), pickle.HIGHEST_PROTOCOL))
use_file = len(data) >= ARG_MAX
if use_file:
with tempfile.NamedTemporaryFile(
delete=False) as the_file:
the_file.write(data)
argv = (ctypes.c_char_p * 5)(sys.executable, __file__,
'@', the_file.name, 0)
else:
argv = (ctypes.c_char_p * 4)(sys.executable, __file__,
data, 0)
pid = ctypes.c_int()
os_environ = (ctypes.c_char_p * (len(os.environ) + 1))(*([
'{0}={1}'.format(*value)
for value in os.environ.iteritems()
] + [0]))
try:
ret = posix_spawn(
ctypes.byref(pid), ctypes.c_char_p(sys.executable),
ctypes.c_void_p(), ctypes.c_void_p(),
ctypes.cast(argv, ctypes.POINTER(ctypes.c_char_p)),
ctypes.cast(os_environ,
ctypes.POINTER(ctypes.c_char_p)))
err = ctypes.get_errno()
os.close(errpipe_write)
if ret:
raise OSError(err, os.strerror(err))
self.pid = pid.value
self._child_created = True
# Wait for exec to fail or succeed; possibly raising exception
# Exception limited to 1M
data = _eintr_retry_call(os.read, errpipe_read,
1048576)
if data:
try:
_eintr_retry_call(os.waitpid, self.pid, 0)
except OSError as e:
if e.errno != errno.ECHILD:
raise
raise pickle.loads(data)
finally:
if use_file:
try:
os.unlink(the_file.name)
except:
pass
os.close(errpipe_read)
if p2cread is not None and p2cwrite is not None:
_close_in_parent(p2cread, to_close)
if c2pwrite is not None and c2pread is not None:
_close_in_parent(c2pwrite, to_close)
if errwrite is not None and errread is not None:
_close_in_parent(errwrite, to_close)
