def close(self):
"""
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
fobj = self.io
if fobj is None:
return
self.io = None
try:
self.flush(_fobj=fobj)
finally:
if self._close:
# Note that we're not using self._apply; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def close(self):
"""
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
fobj = self.io
if fobj is None:
return
self.io = None
try:
self.flush(_fobj=fobj)
finally:
if self._close:
# Note that we're not using self._apply; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged
def close():
try:
fobj.close()
except:
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def close(self):
fobj = self.io
if fobj is None:
return
self.io = None
try:
self.flush(_fobj=fobj)
finally:
if self._close:
# Note that we're not using self._apply; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged
def close():
try:
fobj.close()
except:
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, int):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
address = sockaddr[0]
if not PY3 and isinstance(address, text_type):
address = address.encode('ascii')
if not isinstance(address, string_types):
raise TypeError('sockaddr[0] must be a string, not %s' %
type(address).__name__)
port = sockaddr[1]
if not isinstance(port, int):
raise TypeError('port must be an integer, not %s' % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address,
str(sockaddr[1]),
family=AF_UNSPEC,
socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
family, _socktype, _proto, _name, address = result[0]
if family == AF_INET:
if len(sockaddr) != 2:
raise error("IPv4 sockaddr must be 2 tuple")
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
self.ares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
if PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
service = '0'
return node, service
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, int):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
address = sockaddr[0]
if not PY3 and isinstance(address, text_type):
address = address.encode('ascii')
if not isinstance(address, string_types):
raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__)
port = sockaddr[1]
if not isinstance(port, int):
raise TypeError('port must be an integer, not %s' % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
family, _socktype, _proto, _name, address = result[0]
if family == AF_INET:
if len(sockaddr) != 2:
raise error("IPv4 sockaddr must be 2 tuple")
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
self.ares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
if PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
service = '0'
return node, service
def start_response(self, status, headers, exc_info=None):
if exc_info:
try:
if self.headers_sent:
# Re-raise original exception if headers sent
reraise(*exc_info)
finally:
# Avoid dangling circular ref
exc_info = None
self.code = int(status.split(" ", 1)[0])
self.status = status
self.response_headers = headers
provided_connection = None
self.provided_date = None
self.provided_content_length = None
for header, value in headers:
header = header.lower()
if header == "connection":
provided_connection = value
elif header == "date":
self.provided_date = value
elif header == "content-length":
self.provided_content_length = value
if self.request_version == "HTTP/1.0" and provided_connection is None:
headers.append(("Connection", "close"))
self.close_connection = True
elif provided_connection == "close":
self.close_connection = True
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != "0":
msg = "Invalid Content-Length for %s response: %r (must be absent or zero)" % (
self.code,
self.provided_content_length,
)
if PY3:
msg = msg.encode("latin-1")
raise AssertionError(msg)
return self.write
def start_response(self, status, headers, exc_info=None):
if exc_info:
try:
if self.headers_sent:
# Re-raise original exception if headers sent
reraise(*exc_info)
finally:
# Avoid dangling circular ref
exc_info = None
self.code = int(status.split(' ', 1)[0])
self.status = status
self.response_headers = headers
provided_connection = None
self.provided_date = None
self.provided_content_length = None
for header, value in headers:
header = header.lower()
if header == 'connection':
provided_connection = value
elif header == 'date':
self.provided_date = value
elif header == 'content-length':
self.provided_content_length = value
if self.request_version == 'HTTP/1.0' and provided_connection is None:
headers.append(('Connection', 'close'))
self.close_connection = True
elif provided_connection == 'close':
self.close_connection = True
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != '0':
msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (
self.code, self.provided_content_length)
if PY3:
msg = msg.encode('latin-1')
raise AssertionError(msg)
return self.write
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, int):
raise TypeError('an integer is required')
if not isinstance(sockaddr, tuple):
raise TypeError('getnameinfo() argument 1 must be a tuple')
address = sockaddr[0]
if not PY3 and isinstance(address, text_type):
address = address.encode('ascii')
if not isinstance(address, string_types):
raise TypeError('sockaddr[0] must be a string, not %s' %
type(address).__name__)
port = sockaddr[1]
if not isinstance(port, int):
raise TypeError('port must be an integer, not %s' % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address,
str(sockaddr[1]),
family=AF_UNSPEC,
socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
family, socktype, proto, name, address = result[0]
if family == AF_INET:
if len(sockaddr) != 2:
raise error("IPv4 sockaddr must be 2 tuple")
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
self.ares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
service = '0'
return node, service
def start_response(self, status, headers, exc_info=None):
if exc_info:
try:
if self.headers_sent:
# Re-raise original exception if headers sent
reraise(*exc_info)
finally:
# Avoid dangling circular ref
exc_info = None
self.code = int(status.split(' ', 1)[0])
self.status = status
self.response_headers = headers
provided_connection = None
self.provided_date = None
self.provided_content_length = None
for header, value in headers:
header = header.lower()
if header == 'connection':
provided_connection = value
elif header == 'date':
self.provided_date = value
elif header == 'content-length':
self.provided_content_length = value
if self.request_version == 'HTTP/1.0' and provided_connection is None:
headers.append(('Connection', 'close'))
self.close_connection = True
elif provided_connection == 'close':
self.close_connection = True
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != '0':
msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (self.code, self.provided_content_length)
raise AssertionError(msg)
return self.write
def _getnameinfo(self, sockaddr, flags):
if not isinstance(flags, int):
raise TypeError("an integer is required")
if not isinstance(sockaddr, tuple):
raise TypeError("getnameinfo() argument 1 must be a tuple")
address = sockaddr[0]
if not PY3 and isinstance(address, text_type):
address = address.encode("ascii")
if not isinstance(address, string_types):
raise TypeError("sockaddr[0] must be a string, not %s" % type(address).__name__)
port = sockaddr[1]
if not isinstance(port, int):
raise TypeError("port must be an integer, not %s" % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
raise error("sockaddr resolved to multiple addresses")
family, socktype, proto, name, address = result[0]
if family == AF_INET:
if len(sockaddr) != 2:
raise error("IPv4 sockaddr must be 2 tuple")
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
self.ares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
service = "0"
return node, service
def start_response(self, status, headers, exc_info=None):
# .. versionchanged:: 1.1b5 handle header/status encoding here
if exc_info:
try:
if self.headers_sent:
# Re-raise original exception if headers sent
reraise(*exc_info)
finally:
# Avoid dangling circular ref
exc_info = None
# Pep 3333, "The start_response callable":
# https://www.python.org/dev/peps/pep-3333/#the-start-response-callable
# "Servers should check for errors in the headers at the time
# start_response is called, so that an error can be raised
# while the application is still running." Here, we check the encoding.
# This aids debugging: headers especially are generated programatically
# and an encoding error in a loop or list comprehension yields an opaque
# UnicodeError without any clue which header was wrong.
# Note that this results in copying the header list at this point, not modifying it,
# although we are allowed to do so if needed. This slightly increases memory usage.
response_headers = []
header = None
value = None
try:
for header, value in headers:
if not isinstance(header, str):
raise UnicodeError("The header must be a native string",
header, value)
if not isinstance(value, str):
raise UnicodeError("The value must be a native string",
header, value)
# Either we're on Python 2, in which case bytes is correct, or
# we're on Python 3 and the user screwed up (because it should be a native
# string). In either case, make sure that this is latin-1 compatible. Under
# Python 2, bytes.encode() will take a round-trip through the system encoding,
# which may be ascii, which is not really what we want. However, the latin-1 encoding
# can encode everything except control characters and the block from 0x7F to 0x9F, so
# explicitly round-tripping bytes through the encoding is unlikely to be of much
# benefit, so we go for speed (the WSGI spec specifically calls out allowing the range
# from 0x00 to 0xFF, although the HTTP spec forbids the control characters).
# Note: Some Python 2 implementations, like Jython, may allow non-octet (above 255) values
# in their str implementation; this is mentioned in the WSGI spec, but we don't
# run on any platform like that so we can assume that a str value is pure bytes.
response_headers.append(
(header if not PY3 else header.encode("latin-1"),
value if not PY3 else value.encode("latin-1")))
except UnicodeEncodeError:
# If we get here, we're guaranteed to have a header and value
raise UnicodeError("Non-latin1 header", header, value)
# Same as above
if not isinstance(status, str):
raise UnicodeError("The status string must be a native string")
# don't assign to anything until the validation is complete, including parsing the
# code
code = int(status.split(' ', 1)[0])
self.status = status if not PY3 else status.encode("latin-1")
self._orig_status = status # Preserve the native string for logging
self.response_headers = response_headers
self.code = code
provided_connection = None
self.provided_date = None
self.provided_content_length = None
for header, value in headers:
header = header.lower()
if header == 'connection':
provided_connection = value
elif header == 'date':
self.provided_date = value
elif header == 'content-length':
self.provided_content_length = value
if self.request_version == 'HTTP/1.0' and provided_connection is None:
response_headers.append((b'Connection', b'close'))
self.close_connection = True
elif provided_connection == 'close':
self.close_connection = True
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != '0':
msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (
self.code, self.provided_content_length)
if PY3:
msg = msg.encode('latin-1')
raise AssertionError(msg)
return self.write
def _raise_exception(self):
reraise(*self.exc_info)
def __init__(self,
args,
bufsize=None,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
shell=False,
cwd=None,
env=None,
universal_newlines=False,
startupinfo=None,
creationflags=0,
threadpool=None,
**kwargs):
"""Create new Popen instance."""
if not PY3 and kwargs:
raise TypeError("Got unexpected keyword arguments", kwargs)
pass_fds = kwargs.pop('pass_fds', ())
start_new_session = kwargs.pop('start_new_session', False)
restore_signals = kwargs.pop('restore_signals', True)
hub = get_hub()
if bufsize is None:
# bufsize has different defaults on Py3 and Py2
if PY3:
bufsize = -1
else:
bufsize = 0
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
any_stdio_set = (stdin is not None or stdout is not None
or stderr is not None)
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError(
"close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
# close_fds has different defaults on Py3/Py2
if PY3:
close_fds = True
else:
close_fds = False
if pass_fds and not close_fds:
import warnings
warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite, c2pread, c2pwrite, errread,
errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._translate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io,
write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
# NOTE: Universal Newlines are broken on Windows/Py3, at least
# in some cases. This is true in the stdlib subprocess module
# as well; the following line would fix the test cases in
# test__subprocess.py that depend on python_universal_newlines,
# but would be inconsistent with the stdlib:
#msvcrt.setmode(self.stdout.fileno(), os.O_TEXT)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout.io.mode = 'r'
self.stdout._translate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._translate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell, p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite,
restore_signals, start_new_session)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def start_response(self, status, headers, exc_info=None):
# .. versionchanged:: 1.1b5 handle header/status encoding here
if exc_info:
try:
if self.headers_sent:
# Re-raise original exception if headers sent
reraise(*exc_info)
finally:
# Avoid dangling circular ref
exc_info = None
# Pep 3333, "The start_response callable":
# https://www.python.org/dev/peps/pep-3333/#the-start-response-callable
# "Servers should check for errors in the headers at the time
# start_response is called, so that an error can be raised
# while the application is still running." Here, we check the encoding.
# This aids debuging: headers especially are generated programatically
# and an encoding error in a loop or list comprehension yields an opaque
# UnicodeError without any clue which header was wrong.
# Note that this results in copying the header list at this point, not modifying it,
# although we are allowed to do so if needed. This slightly increases memory usage.
response_headers = []
header = None
value = None
try:
for header, value in headers:
if not isinstance(header, str):
raise UnicodeError("The header must be a native string", header, value)
if not isinstance(value, str):
raise UnicodeError("The value must be a native string", header, value)
# Either we're on Python 2, in which case bytes is correct, or
# we're on Python 3 and the user screwed up (because it should be a native
# string). In either case, make sure that this is latin-1 compatible. Under
# Python 2, bytes.encode() will take a round-trip through the system encoding,
# which may be ascii, which is not really what we want. However, the latin-1 encoding
# can encode everything except control characters and the block from 0x7F to 0x9F, so
# explicitly round-tripping bytes through the encoding is unlikely to be of much
# benefit, so we go for speed (the WSGI spec specifically calls out allowing the range
# from 0x00 to 0xFF, although the HTTP spec forbids the control characters).
# Note: Some Python 2 implementations, like Jython, may allow non-octet (above 255) values
# in their str implementation; this is mentioned in the WSGI spec, but we don't
# run on any platform like that so we can assume that a str value is pure bytes.
response_headers.append((header if not PY3 else header.encode("latin-1"),
value if not PY3 else value.encode("latin-1")))
except UnicodeEncodeError:
# If we get here, we're guaranteed to have a header and value
raise UnicodeError("Non-latin1 header", header, value)
# Same as above
if not isinstance(status, str):
raise UnicodeError("The status string must be a native string")
# don't assign to anything until the validation is complete, including parsing the
# code
code = int(status.split(' ', 1)[0])
self.status = status if not PY3 else status.encode("latin-1")
self.response_headers = response_headers
self.code = code
provided_connection = None
self.provided_date = None
self.provided_content_length = None
for header, value in headers:
header = header.lower()
if header == 'connection':
provided_connection = value
elif header == 'date':
self.provided_date = value
elif header == 'content-length':
self.provided_content_length = value
if self.request_version == 'HTTP/1.0' and provided_connection is None:
response_headers.append((b'Connection', b'close'))
self.close_connection = True
elif provided_connection == 'close':
self.close_connection = True
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != '0':
msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (self.code, self.provided_content_length)
if PY3:
msg = msg.encode('latin-1')
raise AssertionError(msg)
return self.write
def _raise_exception(self):
if self._exc_info:
reraise(self._exc_info[0], self._exc_info[1], load_traceback(self._exc_info[2]))
raise self._exception
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
# XXX: On Python 3, we don't implement these keyword arguments:
# (see patched_tests_setup)
# - pass_fds,
# - start_new_session,
# - restore_signals
hub = get_hub()
if bufsize is None and PY3:
bufsize = -1 # restore default
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._tranlate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout._tranlate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._tranlate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def _raise_exception(self):
if self._exc_info:
reraise(self._exc_info[0], self._exc_info[1], load_traceback(self._exc_info[2]))
raise self._exception
def __init__(self, args, bufsize=None, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None,
**kwargs):
"""Create new Popen instance."""
if not PY3 and kwargs:
raise TypeError("Got unexpected keyword arguments", kwargs)
pass_fds = kwargs.pop('pass_fds', ())
start_new_session = kwargs.pop('start_new_session', False)
restore_signals = kwargs.pop('restore_signals', True)
hub = get_hub()
if bufsize is None:
# bufsize has different defaults on Py3 and Py2
if PY3:
bufsize = -1
else:
bufsize = 0
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
any_stdio_set = (stdin is not None or stdout is not None or
stderr is not None)
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
# close_fds has different defaults on Py3/Py2
if PY3:
close_fds = True
else:
close_fds = False
if pass_fds and not close_fds:
import warnings
warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._translate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
# NOTE: Universal Newlines are broken on Windows/Py3, at least
# in some cases. This is true in the stdlib subprocess module
# as well; the following line would fix the test cases in
# test__subprocess.py that depend on python_universal_newlines,
# but would be inconsistent with the stdlib:
#msvcrt.setmode(self.stdout.fileno(), os.O_TEXT)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout.io.mode = 'r'
self.stdout._translate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._translate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def _raise_exception(self):
reraise(*self.exc_info)
def __init__(self,
args,
bufsize=0,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=False,
shell=False,
cwd=None,
env=None,
universal_newlines=False,
startupinfo=None,
creationflags=0,
threadpool=None):
"""Create new Popen instance."""
# XXX: On Python 3, we don't implement these keyword arguments:
# (see patched_tests_setup)
# - pass_fds,
# - start_new_session,
# - restore_signals
hub = get_hub()
if bufsize is None and PY3:
bufsize = -1 # restore default
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None
or stderr is not None):
raise ValueError(
"close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite, c2pread, c2pwrite, errread,
errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._tranlate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io,
write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout._tranlate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._tranlate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds, cwd,
env, universal_newlines, startupinfo,
creationflags, shell, p2cread, p2cwrite,
c2pread, c2pwrite, errread, errwrite)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
