def event_producer_loop(dispatch):
istream = FileObject(sys.stdin)
ostream = FileObject(sys.stdout, mode='w')
while True:
signal(ostream, READY)
event = wait_for_event(istream)
dispatch(event)
signal(ostream, ACKNOWLEDGED)
def test_read1(self):
# Issue #840
r, w = os.pipe()
x = FileObject(r)
y = FileObject(w, 'w')
self._close_on_teardown(x)
self._close_on_teardown(y)
self.assertTrue(hasattr(x, 'read1'))
def test_newlines(self):
r, w = os.pipe()
lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6']
g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
try:
result = FileObject(r, 'rU').read()
self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6', result)
finally:
g.kill()
def _do_test_del(self, pipe, **kwargs):
r, w = pipe
s = FileObject(w, 'wb', **kwargs)
ts = type(s)
s.write(b'x')
try:
s.flush()
except IOError:
# Sometimes seen on Windows/AppVeyor
print("Failed flushing fileobject", repr(s), file=sys.stderr)
import traceback
traceback.print_exc()
del s # Deliberately getting ResourceWarning with FileObject(Thread) under Py3
gc.collect() # PyPy
if kwargs.get("close", True):
try:
os.close(w)
except (OSError, IOError):
pass # expected, because FileObject already closed it
else:
raise AssertionError('os.close(%r) must not succeed on %r' % (w, ts))
else:
os.close(w)
fobj = FileObject(r, 'rb')
self.assertEqual(fobj.read(), b'x')
fobj.close()
def test_newlines(self):
r, w = os.pipe()
lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6']
g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
try:
fobj = FileObject(r, 'rU')
result = fobj.read()
fobj.close()
self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6', result)
finally:
g.kill()
def test_bufsize_0(self):
# Issue #840
r, w = os.pipe()
x = FileObject(r, 'rb', bufsize=0)
y = FileObject(w, 'wb', bufsize=0)
y.write(b'a')
b = x.read(1)
self.assertEqual(b, b'a')
y.writelines([b'2'])
b = x.read(1)
self.assertEqual(b, b'2')
def _test_del(self, **kwargs):
r, w = os.pipe()
s = FileObject(w, 'wb')
s.write('x')
s.flush()
del s
try:
os.close(w)
except OSError:
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
self.assertEqual(FileObject(r).read(), 'x')
def test_close_pipe(self):
# Issue #190, 203
r, w = os.pipe()
x = FileObject(r)
y = FileObject(w, 'w')
x.close()
y.close()
class LogArchiver(BaseLogProcessor):
pool = ThreadPool(3)
def __init__(self, cname, local_dir):
self.cname = cname
self.local_dir = local_dir
self.f = None
self._cur_date = None
self._aof_file = os.path.join(self.local_dir, "LOG")
self._tmp_file = os.path.join(self.local_dir, "TMP")
self._gz_tmpl = os.path.join(self.local_dir, self.cname + "_{date}.gz")
self._open_log()
self.queue = Queue()
gevent.spawn(self._write_log)
def _open_log(self):
self.f = FileObject(open(self._aof_file, "a+"), "a+")
def compress(self, date):
self.f.close()
os.rename(self._aof_file, self._tmp_file)
LogArchiver.pool.spawn(self._compress, date)
self._open_log()
def _compress(self, date):
f_in = open(self._tmp_file, "rb")
filename = self._gz_tmpl.format(date=date)
gz = gzip.open(filename, "wb")
gz.writelines(f_in)
gz.close()
f_in.close()
@classmethod
def join(cls):
cls.pool.join()
def _write_log(self):
while 1:
entry = self.queue.get()
date = entry["t"].date()
if self._cur_date is None:
self._cur_date = entry["t"].date()
if date > self._cur_date:
self.compress(self._cur_date.strftime("%Y-%m-%d"))
self._cur_date = date
self.f.write(entry["msg"])
if not entry["msg"].endswith("\n"):
self.f.write("\n")
self.f.flush()
def push(self, entry):
self.queue.put(entry)
def test_del_noclose(self):
r, w = os.pipe()
s = FileObject(w, 'wb', close=False)
s.write(b'x')
s.flush()
if PYPY:
s.close()
else:
del s
os.close(w)
self.assertEqual(FileObject(r, 'rb').read(), b'x')
def test_read1(self):
# Issue #840
r, w = os.pipe()
x = FileObject(r)
y = FileObject(w, 'w')
assert hasattr(x, 'read1'), x
x.close()
y.close()
def set_listener(self, listener):
"""
Override of :meth:`~gevent.baseserver.BaseServer.set_listener` to
initialize a pty and properly fill the address
"""
if listener is None:
self.master, self.slave = pty.openpty(
) # returns the 2 file descriptors within the current process.
else:
self.master, self.slave = listener
self.address = os.ttyname(
self.slave
) # /dev/pts/N : slave pts to use to communicate with emulator.
self.fileobj = FileObject(
self.master, mode='rb'
) # <FileObjectPosix <SocketAdapter at 0x1723490 (7, 'rb')>>
# Make a link to the randomly named pseudo-terminal with a known name.
link_path, link_fname = os.path.split(self.link_name)
try:
os.remove(self.link_name)
except:
pass
if not os.path.exists(link_path):
os.makedirs(link_path)
os.symlink(self.address, self.link_name)
print(
"Created symbolic link \"%s\" to emulator pseudo terminal \"%s\" "
% (self.link_name, self.address))
def test_del_noclose(self):
r, w = os.pipe()
s = SocketAdapter(w, close=False)
s.sendall('x')
del s
os.close(w)
self.assertEqual(FileObject(r).read(), 'x')
def _run(self):
# UML communicates with us through IP over SLIP on stdin
self.stream = FileObject(open(0, 'rb', buffering=0), bufsize=0)
# Limitation of requiring stream to be buffered... Please duck-type
del sliplib.SlipStream.__init__
self.stream = sliplib.SlipStream(self.stream)
while True:
msg = self.stream.recv_msg()
if not msg:
break
ip_version = msg[0] >> 4
if ip_version == 4:
from pypacker.layer3.ip import IP
packet = IP(msg)
elif ip_version == 6:
from pypacker.layer3.ip6 import IP6
packet = IP6(msg)
# TODO: IP6 support
continue
else:
# IP packet with unsupported version :(
continue
print(packet.bin(), file=__import__('sys').stderr, flush=True)
IPhandler(self.stream, packet).start()
def read(self):
"""
Gets the allowed entries, groups, and roles from the auth file.
:returns: tuple of allow-entries-list, groups-dict, roles-dict
:rtype: tuple"""
try:
create_file_if_missing(self.auth_file)
with open(self.auth_file) as fil:
# Use gevent FileObject to avoid blocking the thread
before_strip_comments = FileObject(fil, close=False).read()
data = strip_comments(before_strip_comments)
if data != before_strip_comments:
_log.warn('Comments in %s are deprecated and will not be '
'preserved', self.auth_file)
if data:
auth_data = jsonapi.loads(data)
else:
auth_data = {}
except Exception:
_log.exception('error loading %s', self.auth_file)
return [], {}, {}
groups = auth_data.get('groups', {})
roles = auth_data.get('roles', {})
entries = self._get_entries(auth_data, groups, roles)
return entries, groups, roles
def _do_test_del(self, pipe, **kwargs):
r, w = pipe
s = FileObject(w, 'wb', **kwargs)
s.write(b'x')
try:
s.flush()
except IOError:
# Sometimes seen on Windows/AppVeyor
print("Failed flushing fileobject", repr(s), file=sys.stderr)
import traceback
traceback.print_exc()
import warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', ResourceWarning)
# Deliberately getting ResourceWarning with FileObject(Thread) under Py3
del s
gc.collect() # PyPy
if kwargs.get("close", True):
with self.assertRaises((OSError, IOError)):
# expected, because FileObject already closed it
os.close(w)
else:
os.close(w)
with FileObject(r, 'rb') as fobj:
self.assertEqual(fobj.read(), b'x')
def test_seek(self):
fileno, path = tempfile.mkstemp('.gevent.test__fileobject.test_seek')
self.addCleanup(os.remove, path)
s = b'a' * 1024
os.write(fileno, b'B' * 15)
os.write(fileno, s)
os.close(fileno)
with open(path, 'rb') as f:
f.seek(15)
native_data = f.read(1024)
with open(path, 'rb') as f_raw:
try:
f = FileObject(f_raw, 'rb')
except ValueError:
# libuv on Travis can raise EPERM
# from FileObjectPosix. I can't produce it on mac os locally,
# don't know what the issue is. This started happening on Jan 19,
# in the branch that caused all watchers to be explicitly closed.
# That shouldn't have any effect on io watchers, though, which were
# already being explicitly closed.
reraiseFlakyTestRaceConditionLibuv()
if PY3 or FileObject is not FileObjectThread:
self.assertTrue(f.seekable())
f.seek(15)
self.assertEqual(15, f.tell())
fileobj_data = f.read(1024)
self.assertEqual(native_data, s)
self.assertEqual(native_data, fileobj_data)
def test_seek(self):
fileno, path = tempfile.mkstemp()
s = b'a' * 1024
os.write(fileno, b'B' * 15)
os.write(fileno, s)
os.close(fileno)
try:
with open(path, 'rb') as f:
f.seek(15)
native_data = f.read(1024)
with open(path, 'rb') as f_raw:
f = FileObject(f_raw, 'rb')
if hasattr(f, 'seekable'):
# Py3
self.assertTrue(f.seekable())
f.seek(15)
self.assertEqual(15, f.tell())
fileobj_data = f.read(1024)
self.assertEqual(native_data, s)
self.assertEqual(native_data, fileobj_data)
finally:
os.remove(path)
def test_del_noclose(self):
r, w = os.pipe()
s = FileObject(w, 'wb', close=False)
s.write('x')
s.flush()
del s
os.close(w)
self.assertEqual(FileObject(r).read(), 'x')
def test_bufsize_0(self):
# Issue #840
r, w = os.pipe()
x = FileObject(r, 'rb', bufsize=0)
y = FileObject(w, 'wb', bufsize=0)
y.write(b'a')
b = x.read(1)
self.assertEqual(b, b'a')
y.writelines([b'2'])
b = x.read(1)
self.assertEqual(b, b'2')
def _test_del(self, **kwargs):
r, w = os.pipe()
s = FileObject(w, 'wb')
s.write(b'x')
s.flush()
if PYPY:
s.close()
else:
del s
try:
os.close(w)
except OSError:
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
self.assertEqual(FileObject(r, 'rb').read(), b'x')
def _read_protected_topics_file(self):
#Read protected topics file and send to router
try:
create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
self._protected_topics = jsonapi.loads(data) if data else {}
self._send_protected_update_to_pubsub(self._protected_topics)
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
def _read(self):
auth_data = {}
try:
create_file_if_missing(self.auth_file)
with open(self.auth_file) as fil:
# Use gevent FileObject to avoid blocking the thread
before_strip_comments = FileObject(fil, close=False).read()
if isinstance(before_strip_comments, bytes):
before_strip_comments = before_strip_comments.decode(
"utf-8")
data = strip_comments(before_strip_comments)
if data:
auth_data = jsonapi.loads(data)
except Exception:
_log.exception('error loading %s', self.auth_file)
allow_list = auth_data.get('allow', [])
groups = auth_data.get('groups', {})
roles = auth_data.get('roles', {})
version = auth_data.get('version', {'major': 0, 'minor': 0})
return allow_list, groups, roles, version
def _test_del(self, **kwargs):
r, w = os.pipe()
s = SocketAdapter(w)
s.sendall('x')
del s
try:
os.close(w)
except OSError:
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
self.assertEqual(FileObject(r).read(), 'x')
def test_seek(self):
fileno, path = tempfile.mkstemp()
s = b'a' * 1024
os.write(fileno, b'B' * 15)
os.write(fileno, s)
os.close(fileno)
try:
with open(path, 'rb') as f:
f.seek(15)
native_data = f.read(1024)
with open(path, 'rb') as f_raw:
f = FileObject(f_raw, 'rb')
if hasattr(f, 'seekable'):
# Py3
self.assertTrue(f.seekable())
f.seek(15)
self.assertEqual(15, f.tell())
fileobj_data = f.read(1024)
self.assertEqual(native_data, s)
self.assertEqual(native_data, fileobj_data)
finally:
os.remove(path)
def test_close_pipe(self):
# Issue #190, 203
r, w = os.pipe()
x = FileObject(r)
y = FileObject(w, 'w')
x.close()
y.close()
def input_loop(s, do_quit, single_cmd):
if not single_cmd:
print('Type quit or a blank line to close the connection')
sys.stdout.flush()
file_in = FileObject(sys.stdin)
while not do_quit.is_set():
sys.stdout.flush()
if single_cmd:
line = single_cmd
else:
line = file_in.readline()
if not line or not line.decode().strip() or re.match('quit$', line.decode().strip().lower()):
print('bye!')
do_quit.set()
break
s.sendall(line)
if single_cmd:
break
def test_read1(self):
# Issue #840
r, w = os.pipe()
x = FileObject(r)
y = FileObject(w, 'w')
assert hasattr(x, 'read1'), x
x.close()
y.close()
def _read_auth_file(self):
auth_path = os.path.join(self.volttron_home, 'auth.json')
try:
with open(auth_path, 'r') as fd:
data = strip_comments(FileObject(fd, close=False).read())
if data:
auth = jsonapi.loads(data)
else:
auth = {}
except IOError:
auth = {}
if 'allow' not in auth:
auth['allow'] = []
return auth, auth_path
def input_loop(s, do_quit, single_cmd):
if not single_cmd:
print('Type quit or a blank line to close the connection')
sys.stdout.flush()
file_in = FileObject(sys.stdin)
while not do_quit.is_set():
sys.stdout.flush()
if single_cmd:
line = single_cmd
else:
line = file_in.readline()
if not line or not line.decode().strip() or re.match(
'quit$',
line.decode().strip().lower()):
print('bye!')
do_quit.set()
break
s.sendall(line)
if single_cmd:
break
def _synchronize_stream(in_stream, out_stream, output=None, show_output=None):
if show_output is None:
show_output = True
try:
in_file = FileObject(in_stream)
for line in in_file:
if show_output:
out_stream.write(line)
if output is not None:
output.append(line.rstrip())
except BaseException as exception:
log.error("Error while reading stream {}: {}".format(repr(in_stream), repr(exception)))
def _read_protected_topics_file(self):
# Read protected topics file and send to router
try:
create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
self._protected_topics = jsonapi.loads(data) if data else {}
if self.core.messagebus == 'rmq':
self._load_protected_topics_for_rmq()
# Deferring the RMQ topic permissions to after "onstart" event
else:
self._send_protected_update_to_pubsub(
self._protected_topics)
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
def _read_platform_address_file(self):
"""
Read the external addresses file
:return:
"""
try:
with open(self._external_address_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
web_addresses = jsonapi.loads(data) if data else {}
return web_addresses
except IOError as e:
_log.error("Error opening file {}".format(self._external_address_file))
raise
except Exception:
_log.exception('error loading %s', self._external_address_file)
raise
def set_listener(self, listener):
if listener is None:
self.master, self.slave = pty.openpty()
else:
self.master, self.slave = listener
self.original_address = os.ttyname(self.slave)
self.fileobj = FileObject(self.master, mode='rb')
# Make a link to the randomly named pseudo-terminal with a known name.
link_path, link_fname = os.path.split(self.address)
try:
os.remove(self.address)
except Exception:
pass
if not os.path.exists(link_path):
os.makedirs(link_path)
os.symlink(self.original_address, self.address)
_log.info('Created symbolic link "%s" to simulator pseudo terminal %r',
self.address, self.original_address)
def _read(self):
auth_data = {}
try:
create_file_if_missing(self.auth_file)
with open(self.auth_file) as fil:
# Use gevent FileObject to avoid blocking the thread
before_strip_comments = FileObject(fil, close=False).read()
data = strip_comments(before_strip_comments)
if data != before_strip_comments:
_log.warn('Comments in %s are deprecated and will not be '
'preserved', self.auth_file)
if data:
auth_data = jsonapi.loads(data)
except Exception:
_log.exception('error loading %s', self.auth_file)
allow_list = auth_data.get('allow', [])
groups = auth_data.get('groups', {})
roles = auth_data.get('roles', {})
version = auth_data.get('version', {'major': 0, 'minor': 0})
return allow_list, groups, roles, version
def read_auth_file(self):
_log.info('loading auth file %s', self.auth_file)
try:
try:
fil = open(self.auth_file)
except IOError as exc:
if exc.errno != errno.ENOENT:
raise
_log.debug('missing auth file %s', self.auth_file)
_log.info('creating auth file %s', self.auth_file)
fd = os.open(self.auth_file, os.O_CREAT | os.O_WRONLY, 0o660)
try:
os.write(fd, _SAMPLE_AUTH_FILE)
finally:
os.close(fd)
self.auth_entries = []
with open(self.auth_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = strip_comments(
FileObject(fil, close=False).read().decode("utf-8"))
auth_data = jsonapi.loads(data)
except Exception:
_log.exception('error loading %s', self.auth_file)
else:
try:
allowed = auth_data['allow']
except KeyError:
_log.warn("missing 'allow' key in auth file %s",
self.auth_file)
allowed = []
entries = []
for entry in allowed:
try:
entries.append(AuthEntry(**entry))
except TypeError:
_log.warn('invalid entry %r in auth file %s', entry,
self.auth_file)
self.auth_entries = entries
_log.info('auth file %s loaded', self.auth_file)
def _read_protected_topics_file(self):
_log.info('loading protected-topics file %s',
self._protected_topics_file)
try:
utils.create_file_if_missing(self._protected_topics_file)
with open(self._protected_topics_file) as fil:
# Use gevent FileObject to avoid blocking the thread
data = FileObject(fil, close=False).read()
topics_data = jsonapi.loads(data) if data else {}
except Exception:
_log.exception('error loading %s', self._protected_topics_file)
else:
write_protect = topics_data.get('write-protect', [])
topics = ProtectedPubSubTopics()
try:
for entry in write_protect:
topics.add(entry['topic'], entry['capabilities'])
except KeyError:
_log.exception('invalid format for protected topics '
'file {}'.format(self._protected_topics_file))
else:
self.vip.pubsub.protected_topics = topics
_log.info('protected-topics file %s loaded',
self._protected_topics_file)
def test_newlines(self):
import warnings
r, w = os.pipe()
lines = [
b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6'
]
g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
# U is deprecated in Python 3, shows up on FileObjectThread
fobj = FileObject(r, 'rU')
result = fobj.read()
fobj.close()
self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6',
result)
finally:
g.kill()
def async_download_file(out_dir, f):
size_skip = 100000000 # > 100mb for noita is sound and shit
names_skip = ["neverskip"] #[".png", "translations", "fonts", "audio"]
#LOG.info("Handling {}".format(f))
#for f in all_files:
f_path = out_dir / Path(f.filename)
if f.is_directory:
f_path.mkdir(parents=True, exist_ok=True)
else:
# handle data/test/file.txt where test doesn't exist yet
if not f_path.parent.exists():
f_path.parent.mkdir(parents=True, exist_ok=True)
#print(f_path)
# flush it down unless we're skipping this type
if not any([(n in f.filename) for n in names_skip]):
with open(f_path, 'wb') as fout:
fout = FileObject(fout)
if size_skip > f.size:
fout.write(f.read())
else:
fout.write(b"")
gevent.sleep()
return True
def __init__(self, args, bufsize=None, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None,
**kwargs):
"""Create new Popen instance."""
if not PY3 and kwargs:
raise TypeError("Got unexpected keyword arguments", kwargs)
pass_fds = kwargs.pop('pass_fds', ())
start_new_session = kwargs.pop('start_new_session', False)
restore_signals = kwargs.pop('restore_signals', True)
hub = get_hub()
if bufsize is None:
# bufsize has different defaults on Py3 and Py2
if PY3:
bufsize = -1
else:
bufsize = 0
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
any_stdio_set = (stdin is not None or stdout is not None or
stderr is not None)
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
# close_fds has different defaults on Py3/Py2
if PY3:
close_fds = True
else:
close_fds = False
if pass_fds and not close_fds:
import warnings
warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._translate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
# NOTE: Universal Newlines are broken on Windows/Py3, at least
# in some cases. This is true in the stdlib subprocess module
# as well; the following line would fix the test cases in
# test__subprocess.py that depend on python_universal_newlines,
# but would be inconsistent with the stdlib:
#msvcrt.setmode(self.stdout.fileno(), os.O_TEXT)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout.io.mode = 'r'
self.stdout._translate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._translate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
# XXX: On Python 3, we don't implement these keyword arguments:
# (see patched_tests_setup)
# - pass_fds,
# - start_new_session,
# - restore_signals
hub = get_hub()
if bufsize is None and PY3:
bufsize = -1 # restore default
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._tranlate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout._tranlate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._tranlate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def _open_log(self):
self.f = FileObject(open(self._aof_file, "a+"), "a+")
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (POSIX version)"""
if isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork()
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
os.close(errwrite)
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if PY3 and isinstance(args, (str, bytes)):
args = [args]
elif not PY3 and isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork_and_watch(self._on_child, self._loop, True, fork)
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
# Close all other fds, if asked for. This must be done
# after preexec_fn runs.
if close_fds:
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self._close_fds(fds_to_keep)
elif hasattr(os, 'get_inheritable'):
# close_fds was false, and we're on
# Python 3.4 or newer, so "all file
# descriptors except standard streams
# are closed, and inheritable handles
# are only inherited if the close_fds
# parameter is False."
for i in xrange(3, MAXFD):
try:
if i == errpipe_write or os.get_inheritable(i):
continue
os.close(i)
except:
pass
if restore_signals:
# restore the documented signals back to sig_dfl;
# not all will be defined on every platform
for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ':
sig = getattr(signal, sig, None)
if sig is not None:
signal.signal(sig, signal.SIG_DFL)
if start_new_session:
os.setsid()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._child_created = True
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite is not None and errread is not None and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
class Popen(object):
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
hub = get_hub()
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = FileObject(p2cwrite, 'wb')
if c2pread is not None:
if universal_newlines:
self.stdout = FileObject(c2pread, 'rU')
else:
self.stdout = FileObject(c2pread, 'rb')
if errread is not None:
if universal_newlines:
self.stderr = FileObject(errread, 'rU')
else:
self.stderr = FileObject(errread, 'rb')
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def communicate(self, input=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr)."""
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
if self.stdout:
stdout = spawn(self.stdout.read)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
stderr = spawn(self.stderr.read)
greenlets.append(stderr)
else:
stderr = None
joinall(greenlets)
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
self.wait()
return (None if stdout is None else stdout.value or '',
None if stderr is None else stderr.value or '')
def poll(self):
return self._internal_poll()
if mswindows:
#
# Windows methods
#
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = CreatePipe(None, 0)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = CreatePipe(None, 0)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = CreatePipe(None, 0)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _make_inheritable(self, handle):
"""Return a duplicate of handle, which is inheritable"""
return DuplicateHandle(GetCurrentProcess(),
handle, GetCurrentProcess(), 0, 1,
DUPLICATE_SAME_ACCESS)
def _find_w9xpopen(self):
"""Find and return absolut path to w9xpopen.exe"""
w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
# Eeek - file-not-found - possibly an embedding
# situation - see if we can locate it in sys.exec_prefix
w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
raise RuntimeError("Cannot locate w9xpopen.exe, which is "
"needed for Popen to work with your "
"shell or platform.")
return w9xpopen
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (MS Windows version)"""
if not isinstance(args, string_types):
args = list2cmdline(args)
# Process startup details
if startupinfo is None:
startupinfo = STARTUPINFO()
if None not in (p2cread, c2pwrite, errwrite):
startupinfo.dwFlags |= STARTF_USESTDHANDLES
startupinfo.hStdInput = p2cread
startupinfo.hStdOutput = c2pwrite
startupinfo.hStdError = errwrite
if shell:
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
startupinfo.wShowWindow = SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format(comspec, args)
if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com":
# Win9x, or using command.com on NT. We need to
# use the w9xpopen intermediate program. For more
# information, see KB Q150956
# (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
w9xpopen = self._find_w9xpopen()
args = '"%s" %s' % (w9xpopen, args)
# Not passing CREATE_NEW_CONSOLE has been known to
# cause random failures on win9x. Specifically a
# dialog: "Your program accessed mem currently in
# use at xxx" and a hopeful warning about the
# stability of your system. Cost is Ctrl+C wont
# kill children.
creationflags |= CREATE_NEW_CONSOLE
# Start the process
try:
hp, ht, pid, tid = CreateProcess(executable, args,
# no special security
None, None,
int(not close_fds),
creationflags,
env,
cwd,
startupinfo)
except pywintypes.error as e:
# Translate pywintypes.error to WindowsError, which is
# a subclass of OSError. FIXME: We should really
# translate errno using _sys_errlist (or similar), but
# how can this be done from Python?
raise WindowsError(*e.args)
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
# to make sure that no handles to the write end of the
# output pipe are maintained in this process or else the
# pipe will not close when the child process exits and the
# ReadFile will hang.
if p2cread is not None:
p2cread.Close()
if c2pwrite is not None:
c2pwrite.Close()
if errwrite is not None:
errwrite.Close()
# Retain the process handle, but close the thread handle
self._handle = hp
self.pid = pid
ht.Close()
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
self.returncode = GetExitCodeProcess(self._handle)
self.result.set(self.returncode)
return self.returncode
def rawlink(self, callback):
if not self.result.ready() and not self._waiting:
self._waiting = True
Greenlet.spawn(self._wait)
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _blocking_wait(self):
WaitForSingleObject(self._handle, INFINITE)
self.returncode = GetExitCodeProcess(self._handle)
return self.returncode
def _wait(self):
self.threadpool.spawn(self._blocking_wait).rawlink(self.result)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
if not self._waiting:
self._waiting = True
self._wait()
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
"""Send a signal to the process
"""
if sig == signal.SIGTERM:
self.terminate()
elif sig == signal.CTRL_C_EVENT:
os.kill(self.pid, signal.CTRL_C_EVENT)
elif sig == signal.CTRL_BREAK_EVENT:
os.kill(self.pid, signal.CTRL_BREAK_EVENT)
else:
raise ValueError("Unsupported signal: {}".format(sig))
def terminate(self):
"""Terminates the process
"""
TerminateProcess(self._handle, 1)
kill = terminate
else:
#
# POSIX methods
#
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
pass
elif stdin == PIPE:
p2cread, p2cwrite = self.pipe_cloexec()
elif isinstance(stdin, int):
p2cread = stdin
else:
# Assuming file-like object
p2cread = stdin.fileno()
if stdout is None:
pass
elif stdout == PIPE:
c2pread, c2pwrite = self.pipe_cloexec()
elif isinstance(stdout, int):
c2pwrite = stdout
else:
# Assuming file-like object
c2pwrite = stdout.fileno()
if stderr is None:
pass
elif stderr == PIPE:
errread, errwrite = self.pipe_cloexec()
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = stderr
else:
# Assuming file-like object
errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _set_cloexec_flag(self, fd, cloexec=True):
try:
cloexec_flag = fcntl.FD_CLOEXEC
except AttributeError:
cloexec_flag = 1
old = fcntl.fcntl(fd, fcntl.F_GETFD)
if cloexec:
fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
else:
fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
def _remove_nonblock_flag(self, fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK)
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def pipe_cloexec(self):
"""Create a pipe with FDs set CLOEXEC."""
# Pipes' FDs are set CLOEXEC by default because we don't want them
# to be inherited by other subprocesses: the CLOEXEC flag is removed
# from the child's FDs by _dup2(), between fork() and exec().
# This is not atomic: we would need the pipe2() syscall for that.
r, w = os.pipe()
self._set_cloexec_flag(r)
self._set_cloexec_flag(w)
return r, w
def _close_fds(self, but):
if hasattr(os, 'closerange'):
os.closerange(3, but)
os.closerange(but + 1, MAXFD)
else:
for i in xrange(3, MAXFD):
if i == but:
continue
try:
os.close(i)
except:
pass
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (POSIX version)"""
if isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork()
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
os.close(errwrite)
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _handle_exitstatus(self, sts):
if os.WIFSIGNALED(sts):
self.returncode = -os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
self.returncode = os.WEXITSTATUS(sts)
else:
# Should never happen
raise RuntimeError("Unknown child exit status!")
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
"""Send a signal to the process
"""
os.kill(self.pid, sig)
def terminate(self):
"""Terminate the process with SIGTERM
"""
self.send_signal(signal.SIGTERM)
def kill(self):
"""Kill the process with SIGKILL
"""
self.send_signal(signal.SIGKILL)
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
hub = get_hub()
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = FileObject(p2cwrite, 'wb')
if c2pread is not None:
if universal_newlines:
self.stdout = FileObject(c2pread, 'rU')
else:
self.stdout = FileObject(c2pread, 'rb')
if errread is not None:
if universal_newlines:
self.stderr = FileObject(errread, 'rU')
else:
self.stderr = FileObject(errread, 'rb')
class Popen(object):
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
if not isinstance(bufsize, (int, long)):
raise TypeError("bufsize must be an integer")
hub = get_hub()
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = FileObject(p2cwrite, 'wb')
if c2pread is not None:
if universal_newlines:
self.stdout = FileObject(c2pread, 'rU')
else:
self.stdout = FileObject(c2pread, 'rb')
if errread is not None:
if universal_newlines:
self.stderr = FileObject(errread, 'rU')
else:
self.stderr = FileObject(errread, 'rb')
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def communicate(self, input=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr)."""
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
if self.stdout:
stdout = spawn(self.stdout.read)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
stderr = spawn(self.stderr.read)
greenlets.append(stderr)
else:
stderr = None
joinall(greenlets)
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
self.wait()
return (None if stdout is None else stdout.value or '',
None if stderr is None else stderr.value or '')
def poll(self):
return self._internal_poll()
if mswindows:
#
# Windows methods
#
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = CreatePipe(None, 0)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = CreatePipe(None, 0)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = CreatePipe(None, 0)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _make_inheritable(self, handle):
"""Return a duplicate of handle, which is inheritable"""
return DuplicateHandle(GetCurrentProcess(),
handle, GetCurrentProcess(), 0, 1,
DUPLICATE_SAME_ACCESS)
def _find_w9xpopen(self):
"""Find and return absolut path to w9xpopen.exe"""
w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
# Eeek - file-not-found - possibly an embedding
# situation - see if we can locate it in sys.exec_prefix
w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
raise RuntimeError("Cannot locate w9xpopen.exe, which is "
"needed for Popen to work with your "
"shell or platform.")
return w9xpopen
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (MS Windows version)"""
if not isinstance(args, types.StringTypes):
args = list2cmdline(args)
# Process startup details
if startupinfo is None:
startupinfo = STARTUPINFO()
if None not in (p2cread, c2pwrite, errwrite):
startupinfo.dwFlags |= STARTF_USESTDHANDLES
startupinfo.hStdInput = p2cread
startupinfo.hStdOutput = c2pwrite
startupinfo.hStdError = errwrite
if shell:
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
startupinfo.wShowWindow = SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format(comspec, args)
if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com":
# Win9x, or using command.com on NT. We need to
# use the w9xpopen intermediate program. For more
# information, see KB Q150956
# (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
w9xpopen = self._find_w9xpopen()
args = '"%s" %s' % (w9xpopen, args)
# Not passing CREATE_NEW_CONSOLE has been known to
# cause random failures on win9x. Specifically a
# dialog: "Your program accessed mem currently in
# use at xxx" and a hopeful warning about the
# stability of your system. Cost is Ctrl+C wont
# kill children.
creationflags |= CREATE_NEW_CONSOLE
# Start the process
try:
hp, ht, pid, tid = CreateProcess(executable, args,
# no special security
None, None,
int(not close_fds),
creationflags,
env,
cwd,
startupinfo)
except pywintypes.error, e:
def _open_log(self):
self.f = FileObject(open(self._aof_file, 'a+'), 'wb')
class Popen(object):
def __init__(self, args, bufsize=None, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None,
**kwargs):
"""Create new Popen instance."""
if not PY3 and kwargs:
raise TypeError("Got unexpected keyword arguments", kwargs)
pass_fds = kwargs.pop('pass_fds', ())
start_new_session = kwargs.pop('start_new_session', False)
restore_signals = kwargs.pop('restore_signals', True)
hub = get_hub()
if bufsize is None:
# bufsize has different defaults on Py3 and Py2
if PY3:
bufsize = -1
else:
bufsize = 0
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
any_stdio_set = (stdin is not None or stdout is not None or
stderr is not None)
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
# close_fds has different defaults on Py3/Py2
if PY3:
close_fds = True
else:
close_fds = False
if pass_fds and not close_fds:
import warnings
warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._translate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
# NOTE: Universal Newlines are broken on Windows/Py3, at least
# in some cases. This is true in the stdlib subprocess module
# as well; the following line would fix the test cases in
# test__subprocess.py that depend on python_universal_newlines,
# but would be inconsistent with the stdlib:
#msvcrt.setmode(self.stdout.fileno(), os.O_TEXT)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout.io.mode = 'r'
self.stdout._translate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._translate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def _get_devnull(self):
if not hasattr(self, '_devnull'):
self._devnull = os.open(os.devnull, os.O_RDWR)
return self._devnull
_stdout_buffer = None
_stderr_buffer = None
def communicate(self, input=None, timeout=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr).
:keyword timeout: Under Python 2, this is a gevent extension; if
given and it expires, we will raise :class:`gevent.timeout.Timeout`.
Under Python 3, this raises the standard :exc:`TimeoutExpired` exception.
"""
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
# If the timeout parameter is used, and the caller calls back after
# getting a TimeoutExpired exception, we can wind up with multiple
# greenlets trying to run and read from and close stdout/stderr.
# That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'.
# We can't just kill the previous greenlets when a timeout happens,
# though, because we risk losing the output collected by that greenlet
# (and Python 3, where timeout is an official parameter, explicitly says
# that no output should be lost in the event of a timeout.) Instead, we're
# watching for the exception and ignoring it. It's not elegant,
# but it works
if self.stdout:
def _read_out():
try:
data = self.stdout.read()
except RuntimeError:
return
if self._stdout_buffer is not None:
self._stdout_buffer += data
else:
self._stdout_buffer = data
stdout = spawn(_read_out)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
def _read_err():
try:
data = self.stderr.read()
except RuntimeError:
return
if self._stderr_buffer is not None:
self._stderr_buffer += data
else:
self._stderr_buffer = data
stderr = spawn(_read_err)
greenlets.append(stderr)
else:
stderr = None
# If we were given stdin=stdout=stderr=None, we have no way to
# communicate with the child, and thus no greenlets to wait
# on. This is a nonsense case, but it comes up in the test
# case for Python 3.5 (test_subprocess.py
# RunFuncTestCase.test_timeout). Instead, we go directly to
# self.wait
if not greenlets and timeout is not None:
result = self.wait(timeout=timeout)
# Python 3 would have already raised, but Python 2 would not
# so we need to do that manually
if result is None:
from gevent.timeout import Timeout
raise Timeout(timeout)
done = joinall(greenlets, timeout=timeout)
if timeout is not None and len(done) != len(greenlets):
if PY3:
raise TimeoutExpired(self.args, timeout)
from gevent.timeout import Timeout
raise Timeout(timeout)
if self.stdout:
try:
self.stdout.close()
except RuntimeError:
pass
if self.stderr:
try:
self.stderr.close()
except RuntimeError:
pass
self.wait()
stdout_value = self._stdout_buffer
self._stdout_buffer = None
stderr_value = self._stderr_buffer
self._stderr_buffer = None
# XXX: Under python 3 in universal newlines mode we should be
# returning str, not bytes
return (None if stdout is None else stdout_value or b'',
None if stderr is None else stderr_value or b'')
def poll(self):
return self._internal_poll()
if PY3:
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
try: # Flushing a BufferedWriter may raise an error
if self.stdin:
self.stdin.close()
finally:
# Wait for the process to terminate, to avoid zombies.
# JAM: gevent: If the process never terminates, this
# blocks forever.
self.wait()
if mswindows:
#
# Windows methods
#
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
try:
DEVNULL
except NameError:
_devnull = object()
else:
_devnull = DEVNULL
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = CreatePipe(None, 0)
if PY3:
p2cread = Handle(p2cread)
_winapi.CloseHandle(_)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
if PY3:
p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
elif stdin == _devnull:
p2cread = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = CreatePipe(None, 0)
if PY3:
c2pwrite = Handle(c2pwrite)
_winapi.CloseHandle(_)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
if PY3:
c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
elif stdout == _devnull:
c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = CreatePipe(None, 0)
if PY3:
errwrite = Handle(errwrite)
_winapi.CloseHandle(_)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
if PY3:
errread, errwrite = Handle(errread), Handle(errwrite)
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == _devnull:
errwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _make_inheritable(self, handle):
"""Return a duplicate of handle, which is inheritable"""
return DuplicateHandle(GetCurrentProcess(),
handle, GetCurrentProcess(), 0, 1,
DUPLICATE_SAME_ACCESS)
def _find_w9xpopen(self):
"""Find and return absolut path to w9xpopen.exe"""
w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
# Eeek - file-not-found - possibly an embedding
# situation - see if we can locate it in sys.exec_prefix
w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
raise RuntimeError("Cannot locate w9xpopen.exe, which is "
"needed for Popen to work with your "
"shell or platform.")
return w9xpopen
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
unused_restore_signals, unused_start_new_session):
"""Execute program (MS Windows version)"""
assert not pass_fds, "pass_fds not supported on Windows."
if not isinstance(args, string_types):
args = list2cmdline(args)
# Process startup details
if startupinfo is None:
startupinfo = STARTUPINFO()
if None not in (p2cread, c2pwrite, errwrite):
startupinfo.dwFlags |= STARTF_USESTDHANDLES
startupinfo.hStdInput = p2cread
startupinfo.hStdOutput = c2pwrite
startupinfo.hStdError = errwrite
if shell:
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
startupinfo.wShowWindow = SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format(comspec, args)
if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com":
# Win9x, or using command.com on NT. We need to
# use the w9xpopen intermediate program. For more
# information, see KB Q150956
# (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
w9xpopen = self._find_w9xpopen()
args = '"%s" %s' % (w9xpopen, args)
# Not passing CREATE_NEW_CONSOLE has been known to
# cause random failures on win9x. Specifically a
# dialog: "Your program accessed mem currently in
# use at xxx" and a hopeful warning about the
# stability of your system. Cost is Ctrl+C wont
# kill children.
creationflags |= CREATE_NEW_CONSOLE
# Start the process
try:
hp, ht, pid, tid = CreateProcess(executable, args,
# no special security
None, None,
int(not close_fds),
creationflags,
env,
cwd,
startupinfo)
except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError
# Translate pywintypes.error to WindowsError, which is
# a subclass of OSError. FIXME: We should really
# translate errno using _sys_errlist (or similar), but
# how can this be done from Python?
if PY3:
raise # don't remap here
raise WindowsError(*e.args)
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
# to make sure that no handles to the write end of the
# output pipe are maintained in this process or else the
# pipe will not close when the child process exits and the
# ReadFile will hang.
def _close(x):
if x is not None and x != -1:
if hasattr(x, 'Close'):
x.Close()
else:
_winapi.CloseHandle(x)
_close(p2cread)
_close(c2pwrite)
_close(errwrite)
if hasattr(self, '_devnull'):
os.close(self._devnull)
# Retain the process handle, but close the thread handle
self._child_created = True
self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp
self.pid = pid
_winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close()
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
self.returncode = GetExitCodeProcess(self._handle)
self.result.set(self.returncode)
return self.returncode
def rawlink(self, callback):
if not self.result.ready() and not self._waiting:
self._waiting = True
Greenlet.spawn(self._wait)
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _blocking_wait(self):
WaitForSingleObject(self._handle, INFINITE)
self.returncode = GetExitCodeProcess(self._handle)
return self.returncode
def _wait(self):
self.threadpool.spawn(self._blocking_wait).rawlink(self.result)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
if not self._waiting:
self._waiting = True
self._wait()
result = self.result.wait(timeout=timeout)
if PY3 and timeout is not None and not self.result.ready():
raise TimeoutExpired(self.args, timeout)
return result
def send_signal(self, sig):
"""Send a signal to the process
"""
if sig == signal.SIGTERM:
self.terminate()
elif sig == signal.CTRL_C_EVENT:
os.kill(self.pid, signal.CTRL_C_EVENT)
elif sig == signal.CTRL_BREAK_EVENT:
os.kill(self.pid, signal.CTRL_BREAK_EVENT)
else:
raise ValueError("Unsupported signal: {}".format(sig))
def terminate(self):
"""Terminates the process
"""
TerminateProcess(self._handle, 1)
kill = terminate
else:
#
# POSIX methods
#
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
try:
DEVNULL
except NameError:
_devnull = object()
else:
_devnull = DEVNULL
if stdin is None:
pass
elif stdin == PIPE:
p2cread, p2cwrite = self.pipe_cloexec()
elif stdin == _devnull:
p2cread = self._get_devnull()
elif isinstance(stdin, int):
p2cread = stdin
else:
# Assuming file-like object
p2cread = stdin.fileno()
if stdout is None:
pass
elif stdout == PIPE:
c2pread, c2pwrite = self.pipe_cloexec()
elif stdout == _devnull:
c2pwrite = self._get_devnull()
elif isinstance(stdout, int):
c2pwrite = stdout
else:
# Assuming file-like object
c2pwrite = stdout.fileno()
if stderr is None:
pass
elif stderr == PIPE:
errread, errwrite = self.pipe_cloexec()
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == _devnull:
errwrite = self._get_devnull()
elif isinstance(stderr, int):
errwrite = stderr
else:
# Assuming file-like object
errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _set_cloexec_flag(self, fd, cloexec=True):
try:
cloexec_flag = fcntl.FD_CLOEXEC
except AttributeError:
cloexec_flag = 1
old = fcntl.fcntl(fd, fcntl.F_GETFD)
if cloexec:
fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
else:
fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
def _remove_nonblock_flag(self, fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK)
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def pipe_cloexec(self):
"""Create a pipe with FDs set CLOEXEC."""
# Pipes' FDs are set CLOEXEC by default because we don't want them
# to be inherited by other subprocesses: the CLOEXEC flag is removed
# from the child's FDs by _dup2(), between fork() and exec().
# This is not atomic: we would need the pipe2() syscall for that.
r, w = os.pipe()
self._set_cloexec_flag(r)
self._set_cloexec_flag(w)
return r, w
def _close_fds(self, keep):
# `keep` is a set of fds, so we
# use os.closerange from 3 to min(keep)
# and then from max(keep + 1) to MAXFD and
# loop through filling in the gaps.
# Under new python versions, we need to explicitly set
# passed fds to be inheritable or they will go away on exec
if hasattr(os, 'set_inheritable'):
set_inheritable = os.set_inheritable
else:
set_inheritable = lambda i, v: True
if hasattr(os, 'closerange'):
keep = sorted(keep)
min_keep = min(keep)
max_keep = max(keep)
os.closerange(3, min_keep)
os.closerange(max_keep + 1, MAXFD)
for i in xrange(min_keep, max_keep):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
else:
for i in xrange(3, MAXFD):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if PY3 and isinstance(args, (str, bytes)):
args = [args]
elif not PY3 and isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork_and_watch(self._on_child, self._loop, True, fork)
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
# Close all other fds, if asked for. This must be done
# after preexec_fn runs.
if close_fds:
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self._close_fds(fds_to_keep)
elif hasattr(os, 'get_inheritable'):
# close_fds was false, and we're on
# Python 3.4 or newer, so "all file
# descriptors except standard streams
# are closed, and inheritable handles
# are only inherited if the close_fds
# parameter is False."
for i in xrange(3, MAXFD):
try:
if i == errpipe_write or os.get_inheritable(i):
continue
os.close(i)
except:
pass
if restore_signals:
# restore the documented signals back to sig_dfl;
# not all will be defined on every platform
for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ':
sig = getattr(signal, sig, None)
if sig is not None:
signal.signal(sig, signal.SIG_DFL)
if start_new_session:
os.setsid()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._child_created = True
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite is not None and errread is not None and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _handle_exitstatus(self, sts):
if os.WIFSIGNALED(sts):
self.returncode = -os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
self.returncode = os.WEXITSTATUS(sts)
else:
# Should never happen
raise RuntimeError("Unknown child exit status!")
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute.
:keyword timeout: The floating point number of seconds to wait.
Under Python 2, this is a gevent extension, and we simply return if it
expires. Under Python 3,
if this time elapses without finishing the process, :exc:`TimeoutExpired`
is raised."""
result = self.result.wait(timeout=timeout)
if PY3 and timeout is not None and not self.result.ready():
raise TimeoutExpired(self.args, timeout)
return result
def send_signal(self, sig):
"""Send a signal to the process
"""
os.kill(self.pid, sig)
def terminate(self):
"""Terminate the process with SIGTERM
"""
self.send_signal(signal.SIGTERM)
def kill(self):
"""Kill the process with SIGKILL
"""
self.send_signal(signal.SIGKILL)
def start(self):
self._reader = Greenlet(self._read_stdin)
self._stdin = FileObject(sys.stdin)
self._stdout = FileObject(sys.stdout)
self._reader.start()
class ConsoleAdapter(object):
"""
This adapter will run Jeev in console mode, listening to stdin for messages,
and writing outgoing messages to stdout.
"""
def __init__(self, jeev, opts):
self._jeev = jeev
self._opts = opts
self._stdin = None
self._stdout = None
self._reader = None
self._channel = opts.get('console_channel', 'console')
self._user = opts.get('console_user', 'user')
def _read_stdin(self):
self._stdout.write(">>> Jeev Console Adapater\n")
self._stdout.write(">>> Switch channel using \c channel_name\n")
self._stdout.write(">>> Switch user using \u user_name\n")
self._stdout.flush()
while True:
self._stdout.write('[%s@%s] > ' % (self._user, self._channel))
self._stdout.flush()
line = self._stdin.readline()
if not line:
break
if line.startswith('\c'):
self._channel = line[2:].strip().lstrip('#')
self._stdout.write("Switched channel to #%s\n" % self._channel)
self._stdout.flush()
elif line.startswith('\u'):
self._user = line[2:].strip()
self._stdout.write("Switched user %s\n" % self._user)
self._stdout.flush()
else:
message = Message({}, self._channel, self._user, line.strip())
self._jeev._handle_message(message)
def start(self):
self._reader = Greenlet(self._read_stdin)
self._stdin = FileObject(sys.stdin)
self._stdout = FileObject(sys.stdout)
self._reader.start()
def stop(self):
self._reader.kill()
self._reader = None
def join(self):
self._reader.join()
def send_message(self, channel, message):
self._stdout.write('\r< [#%s] %s\n' % (channel, message))
self._stdout.write('[%s@%s] > ' % (self._user, self._channel))
self._stdout.flush()
def send_messages(self, channel, *messages):
for message in messages:
self.send_message(channel, message)
def _test_del(self, **kwargs):
r, w = os.pipe()
s = FileObject(w, 'wb')
s.write(b'x')
s.flush()
if PYPY:
s.close()
else:
del s # Deliberately getting ResourceWarning under Py3
try:
os.close(w)
except OSError:
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
fobj = FileObject(r, 'rb')
self.assertEqual(fobj.read(), b'x')
fobj.close()
class LogArchiver(BaseLogProcessor):
pool = ThreadPool(3) # 使得所有LogArchiver类对象能共享线程池
def __init__(self, cname, local_dir):
self.cname = cname
self.local_dir = local_dir
self.f = None
self._cur_date = None
self._aof_file = os.path.join(
self.local_dir, "LOG")
self._tmp_file = os.path.join(
self.local_dir, "TMP")
self._gz_tmpl = os.path.join(
self.local_dir, self.cname + "_{date}.gz") # 存储压缩文件
# 主程序开始
self._open_log()
self.queue = Queue()
gevent.spawn(self._write_log) # 触发一个协程
# 主程序结束
def _open_log(self):
self.f = FileObject(open(self._aof_file, 'a+'), 'wb')
def compress(self, date):
self.f.close()
os.rename(self._aof_file, self._tmp_file)
LogArchiver.pool.spawn(self._compress, date)
self._open_log()
def _compress(self, date): # 如果数据过大则压缩
f_in = open(self._tmp_file, 'rb')
filename = self._gz_tmpl.format(date=date)
gz = gzip.open(filename, 'wb')
gz.writelines(f_in)
gz.close()
f_in.close()
@classmethod
def join(cls):
cls.pool.join()
def _write_log(self):
while 1:
entry = self.queue.get()
date = entry['t'].date()
if self._cur_date is None:
self._cur_date = entry['t'].date()
if date > self._cur_date:
self.compress(self._cur_date.strftime("%Y-%m-%d"))
self._cur_date = date
self.f.write(entry['msg'])
if not entry['msg'].endswith('\n'):
self.f.write('\n')
self.f.flush()
def push(self, entry):
self.queue.put(entry)
