def _check_ready(self, conn, maxsize=1024):
if maxsize < 1:
maxsize = 1
if conn is None:
return
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
if conn is self.stdout:
self.emit('stdout-ready')
elif conn is self.stderr:
self.emit('stderr-ready')
except ValueError:
return conn.close()
except (WindowsError, Exception) as ex:
if ex[0] in (109, errno.ESHUTDOWN):
return conn.close()
raise
return True
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
# except (subprocess.pywintypes.error, Exception):
except pywintypes.error:
why = sys.exc_info()[1]
if why.winerror in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read, self._encoding)
else:
read = read.decode(self._encoding)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception) as why:
if why.args[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
getattr(self, '{0}_buff'.format(which)).write(read)
getattr(self, '_{0}_logger'.format(which)).debug(read.rstrip())
if self.stream_stds:
getattr(sys, which).write(read)
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception):
if geterror()[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
# Translate newlines. For Python 3.x assume read is text.
# If bytes then another solution is needed.
## read = self._translate_newlines(read)
read = read.replace("\r\n", "\n").replace("\r", "\n")
return read
def _recv(self, which, maxsize):
'''Private method for receiving data from process
Usage: instance( which, maxsize (
which - connection to receive output from
maxsize - maximm size of buffer to be received'''
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception) as why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
if not hasattr(self, '%sHandle' % which):
setattr(self, '%sHandle' % which,
msvcrt.get_osfhandle(conn.fileno()))
Handle = getattr(self, '%sHandle' % which)
read, nAvail, nMessage = PeekNamedPipe(Handle, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
errCode, read = ReadFile(Handle, nAvail, None)
except ValueError:
return self._close(which)
except SystemExit:
raise
except (subprocess.pywintypes.error, Exception) as why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines and False:
read = self._translate_newlines(read)
return read
def can_read(self):
if subprocess.mswindows:
from msvcrt import get_osfhandle
from win32pipe import PeekNamedPipe
handle = get_osfhandle(self.proc.stdout.fileno())
return PeekNamedPipe(handle, 0)[2] != 0
else:
return _fileno_can_read(self.proc.stdout.fileno())
def nonblockrecv(conn, maxsize):
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
return read
def non_block_read_line(obj): try: handle = get_osfhandle(obj.fileno()) (read, nAvail, nMessage) = PeekNamedPipe(handle, 0) if nAvail > 0: return obj.readline() except: pass return ""
def can_read(self):
if sys.platform == 'win32':
from msvcrt import get_osfhandle
from win32pipe import PeekNamedPipe
handle = get_osfhandle(self.proc.stdout.fileno())
data, total_bytes_avail, msg_bytes_left = PeekNamedPipe(handle, 0)
return total_bytes_avail != 0
else:
return _fileno_can_read(self.proc.stdout.fileno())
def pipe_read( self, _pipe, _minimum_to_read ):
## Hackaround since Windows doesn't support select() except for sockets.
dbg_print( 'pipe_read: minimum to read is ' + str( _minimum_to_read ) )
dbg_print( 'pipe_read: sleeping for ' + str( self.delay ) + ' seconds' )
time.sleep( self.delay )
data = ''
osfh = msvcrt.get_osfhandle( _pipe )
( read, count, msg ) = PeekNamedPipe( osfh, 0 )
dbg_print( 'pipe_read: initial count via PeekNamedPipe is ' + str( count ) )
while ( count > 0 ):
dbg_print( 'pipe_read: reading from pipe' )
dbg_print( '' )
( err, tmp ) = ReadFile( osfh, count, None )
data += tmp
dbg_print( 'pipe_read: read ' + str( tmp ) )
( read, count, msg ) = PeekNamedPipe( osfh, 0 )
## Be sure to break the read, if asked to do so,
## after we've read in a line termination.
if _minimum_to_read != 0 and len( data ) > 0 and data[ len( data ) -1 ] == '\n':
if len( data ) >= _minimum_to_read:
dbg_print( 'pipe_read: found termination and read at least the minimum asked for' )
break
else:
dbg_print( 'pipe_read: not all of the data has been read' )
dbg_print( 'pipe_read: returning' )
return data
def _mergedReader(self):
noop = []
handles = self._handles
while handles:
if _mswindows:
new_data = None
for handle in list(handles):
try:
pipe = get_osfhandle(handle.read_pipe)
numAvail = PeekNamedPipe(pipe, 0)[1]
if numAvail:
result, new_data = ReadFile(pipe, numAvail, None)
handle.decoder_buffer += new_data
break
except:
handle.finalize(self.ostreams)
handles.remove(handle)
new_data = None
if new_data is None:
# PeekNamedPipe is non-blocking; to avoid swamping
# the core, sleep for a "short" amount of time
time.sleep(_poll_interval)
continue
else:
# Because we could be *adding* handles to the TeeStream
# while the _mergedReader is running, we want to
# periodically time out and update the list of handles
# that we are waiting for. It is also critical that we
# send select() a *copy* of the handles list, as we see
# deadlocks when handles are added while select() is
# waiting
ready_handles = select(
list(handles), noop, noop, _poll_interval)[0]
if not ready_handles:
continue
handle = ready_handles[0]
new_data = os.read(handle.read_pipe, io.DEFAULT_BUFFER_SIZE)
if not new_data:
handle.finalize(self.ostreams)
handles.remove(handle)
continue
handle.decoder_buffer += new_data
# At this point, we have new data sitting in the
# handle.decoder_buffer
handle.decodeIncomingBuffer()
# Now, output whatever we have decoded to the output streams
handle.writeOutputBuffer(self.ostreams)
def _state(self, which):
conn, maxsize = self.get_conn_maxsize(which, None)
if conn is None:
return PIPE_CLOSED
try:
x = msvcrt.get_osfhandle(conn.fileno()) # @UndefinedVariable
except:
return PIPE_CLOSED
try:
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
except:
return PIPE_CLOSED
if nAvail > 0:
return PIPE_READY2READ
return PIPE_EMPTY
def _recv(self, which, maxsize = 32768):
conn = getattr(self, which)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception), why:
if why[0] in (109, ESHUTDOWN):
return self._close(which)
print "Error: Shell no stdout"
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception), why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
def __recv(logger, conn, maxsize = 1024*512): try: x = msvcrt.get_osfhandle(conn.fileno()) (read, nAvail, nMessage) = PeekNamedPipe(x, 0) if maxsize < nAvail: nAvail = maxsize if nAvail > 0: (errCode, read) = ReadFile(x, nAvail, None) #read = subprocess.Popen._translate_newlines(read) except ValueError: logger.error( "get one unknow error when receive data" ) return '', "unknown error" except (subprocess.pywintypes.error, Exception), why: if why[0] in (109, errno.ESHUTDOWN): if why[0] != 109: logger.error( "get error when read info from pipe %s, pipe id:%d, reason:%s"%(name, which.pid, str(why)) ) return '', str(why[-1]) return '', str(why[-1])
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno()) # @UndefinedVariable
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except:
return None
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except (ValueError, NameError):
return self._close(which)
except (subprocess.pywintypes.error, Exception) as ex:
if ex[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, _) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(_, read) = ReadFile(x, nAvail, None)
except (ValueError, NameError):
return self._close(which)
except Exception as ex:
if getattr(ex, "args",
None) and ex.args[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _serverStdOutHasData(self, maxsize=256):
"""
Used by startServer pipe reader code. Allows for async check for data on pipe in windows.
"""
# >> WIN32_ONLY
import msvcrt
from win32pipe import PeekNamedPipe
maxsize = self._get_maxsize(maxsize)
conn = self._server_process.stdout
if conn is None:
return False
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
return True
# << WIN32_ONLY
except Exception as e:
raise ioHubClientException(e)
def _mergedReader(self):
noop = []
handles = self._active_handles
_poll = _poll_interval
_fast_poll_ct = _poll_rampup
new_data = '' # something not None
while handles:
if new_data is None:
# For performance reasons, we use very aggressive
# polling at the beginning (_poll_interval) and then
# ramp up to a much more modest polling interval
# (_poll_rampup_limit) as the process runs and the
# frequency of new data appearing on the pipe slows
if _fast_poll_ct:
_fast_poll_ct -= 1
if not _fast_poll_ct:
_poll *= 10
if _poll < _poll_rampup_limit:
# reset the counter (to potentially increase
# the polling interval again)
_fast_poll_ct = _poll_rampup
else:
new_data = None
if _mswindows:
for handle in list(handles):
try:
pipe = get_osfhandle(handle.read_pipe)
numAvail = PeekNamedPipe(pipe, 0)[1]
if numAvail:
result, new_data = ReadFile(pipe, numAvail, None)
handle.decoder_buffer += new_data
break
except:
handles.remove(handle)
new_data = None
if new_data is None:
# PeekNamedPipe is non-blocking; to avoid swamping
# the core, sleep for a "short" amount of time
time.sleep(_poll)
continue
else:
# Because we could be *adding* handles to the TeeStream
# while the _mergedReader is running, we want to
# periodically time out and update the list of handles
# that we are waiting for. It is also critical that we
# send select() a *copy* of the handles list, as we see
# deadlocks when handles are added while select() is
# waiting
ready_handles = select(list(handles), noop, noop, _poll)[0]
if not ready_handles:
new_data = None
continue
handle = ready_handles[0]
new_data = os.read(handle.read_pipe, io.DEFAULT_BUFFER_SIZE)
if not new_data:
handles.remove(handle)
continue
handle.decoder_buffer += new_data
# At this point, we have new data sitting in the
# handle.decoder_buffer
handle.decodeIncomingBuffer()
# Now, output whatever we have decoded to the output streams
handle.writeOutputBuffer(self.ostreams)
def _merged_reader(*args):
class StreamData(object):
__slots__ = ('read', 'output', 'unbuffer', 'buf', 'data')
def __init__(self, *args):
if _mswindows:
self.read = get_osfhandle(args[1])
else:
self.read = args[1]
self.unbuffer = args[0]
self.output = tuple(x for x in args[2:] if x is not None)
self.buf = ""
self.data = None
def write(self, x):
success = True
for s in self.output:
try:
s.write(x)
except ValueError:
success = False
return success
def flush(self):
for s in self.output:
try:
s.flush()
except ValueError:
pass
raw_stderr = sys.__stderr__
if raw_stderr is None:
# There are cases, e.g., in Anaconda, where there is no stdout
# for the original process because, for example, it was started
# in a windowing environment.
raw_stderr = sys.stderr
try:
encoding = stream.encoding
except:
encoding = None
if encoding is None:
try:
encoding = raw_stderr.encoding
except:
pass
if encoding is None:
encoding = 'utf-8'
streams = {}
for s in args:
tmp = StreamData(*s)
streams[tmp.read] = tmp
handles = sorted(streams.keys(), key=lambda x: -1 * streams[x].unbuffer)
noop = []
while handles:
if _mswindows:
new_data = None
for h in handles:
try:
numAvail = PeekNamedPipe(h, 0)[1]
if numAvail == 0:
continue
result, new_data = ReadFile(h, 1, None)
except:
handles.remove(h)
new_data = None
if new_data is None:
continue
else:
h = select(handles, noop, noop)[0]
if not h:
break
h = h[0]
new_data = os.read(h, 1)
if not new_data:
handles.remove(h)
continue
s = streams[h]
if s.data is None:
s.data = new_data
else:
s.data += new_data
char = s.data.decode(encoding)
if char.encode(encoding) != s.data:
continue
s.data = None
if s.unbuffer:
writeOK = s.write(char)
s.buf += char
if char[-1] != "\n":
continue
if s.unbuffer:
s.flush()
else:
writeOK = s.write(s.buf)
if writeOK:
s.buf = ""
writeOK = True
for s in itervalues(streams):
if s.buf:
writeOK &= s.write(s.buf)
if s.data:
writeOK &= s.write(s.data.decode(encoding))
s.flush()
if not writeOK and raw_stderr is not None:
raw_stderr.write("""
ERROR: pyutilib.subprocess: output stream closed before all subprocess output
was written to it. The following was left in the subprocess buffer:
'%s'
""" % (buf, ))
if data:
raw_stderr.write(
"""The following undecoded unicode output was also present:
'%s'
""" % (data, ))
