def get_n_packets(process):
"""
@param process the process helper created with run_helper
@return the number of packets that the helper prints on the stderr >= 0
negative on errors.
"""
if not WINDOWS:
try:
inp, out, err = select.select([process.stderr], [process.stderr],
[process.stderr])
except:
# Here we could have select that hangs after a kill in stop
return -1
if process.stderr in inp:
line = process.stderr.read()
if not line:
return -2
else:
try:
# PeekNamedPipe is async here and doesn't block resulting
# in gui freeze. So we use directly ReadFile here
x = msvcrt.get_osfhandle(process.stderr.fileno())
errCode, line = ReadFile(x, 1024, None)
except:
return -2
# Here dumpcap use '\rPackets: %u ' while tcpdump 'Got %u\r'
# over stderr file. We use simple split(' ')[1]
try:
return int(line.split(' ')[1])
except:
return -3
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception) as why:
if why.args[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
getattr(self, '{0}_buff'.format(which)).write(read)
getattr(self, '_{0}_logger'.format(which)).debug(read.rstrip())
if self.stream_stds:
getattr(sys, which).write(read)
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
'''Private method for receiving data from process
Usage: instance( which, maxsize (
which - connection to receive output from
maxsize - maximm size of buffer to be received'''
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception) as why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
if not hasattr(self, '%sHandle' % which):
setattr(self, '%sHandle' % which,
msvcrt.get_osfhandle(conn.fileno()))
Handle = getattr(self, '%sHandle' % which)
read, nAvail, nMessage = PeekNamedPipe(Handle, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
errCode, read = ReadFile(Handle, nAvail, None)
except ValueError:
return self._close(which)
except SystemExit:
raise
except (subprocess.pywintypes.error, Exception) as why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines and False:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception):
if geterror()[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
# Translate newlines. For Python 3.x assume read is text.
# If bytes then another solution is needed.
## read = self._translate_newlines(read)
read = read.replace("\r\n", "\n").replace("\r", "\n")
return read
def _read_header(handle, bufsize=4096):
"""INTERNAL: read a stub header from a handle."""
header = ''
while '\n\n' not in header:
err, data = ReadFile(handle, bufsize)
header += data
return header
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
# except (subprocess.pywintypes.error, Exception):
except pywintypes.error:
why = sys.exc_info()[1]
if why.winerror in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read, self._encoding)
else:
read = read.decode(self._encoding)
return read
def nonblockrecv(conn, maxsize):
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
return read
def _pipe_read(self, buf):
data = []
hr = winerror.ERROR_MORE_DATA
while hr==winerror.ERROR_MORE_DATA:
hr, d = ReadFile(self.pipe_handle, 65536)
data.append(d)
s = b"".join(data)
log("pipe_read: %i / %s", hr, binascii.hexlify(s))
return s
def read(self, amount=8192):
if self._closed:
return False
try:
error, data = ReadFile(self._conout_pipe, amount)
except:
data = None
return data
def ReceiveThread(self):
from win32event import (
ResetEvent,
MsgWaitForMultipleObjects,
QS_ALLINPUT,
WAIT_OBJECT_0,
WAIT_TIMEOUT,
)
from win32file import ReadFile, AllocateReadBuffer, GetOverlappedResult
from win32api import GetLastError
continueLoop = True
overlapped = self.serial._overlappedRead
hComPort = self.serial.hComPort
hEvent = overlapped.hEvent
stopEvent = self.stopEvent
n = 1
waitingOnRead = False
buf = AllocateReadBuffer(n)
while continueLoop:
if not waitingOnRead:
ResetEvent(hEvent)
hr, _ = ReadFile(hComPort, buf, overlapped)
if hr == 997:
waitingOnRead = True
elif hr == 0:
pass
#n = GetOverlappedResult(hComPort, overlapped, 1)
#self.HandleChar(str(buf))
else:
self.PrintError("error")
raise
rc = MsgWaitForMultipleObjects(
(hEvent, stopEvent),
0,
1000,
QS_ALLINPUT
)
if rc == WAIT_OBJECT_0:
n = GetOverlappedResult(hComPort, overlapped, 1)
if n:
self.HandleChar(str(buf))
#else:
# print "WAIT_OBJECT_0", n, str(buf[:n])
waitingOnRead = False
elif rc == WAIT_OBJECT_0+1:
continueLoop = False
elif rc == WAIT_TIMEOUT:
pass
else:
self.PrintError("unknown message")
def _mergedReader(self):
noop = []
handles = self._handles
while handles:
if _mswindows:
new_data = None
for handle in list(handles):
try:
pipe = get_osfhandle(handle.read_pipe)
numAvail = PeekNamedPipe(pipe, 0)[1]
if numAvail:
result, new_data = ReadFile(pipe, numAvail, None)
handle.decoder_buffer += new_data
break
except:
handle.finalize(self.ostreams)
handles.remove(handle)
new_data = None
if new_data is None:
# PeekNamedPipe is non-blocking; to avoid swamping
# the core, sleep for a "short" amount of time
time.sleep(_poll_interval)
continue
else:
# Because we could be *adding* handles to the TeeStream
# while the _mergedReader is running, we want to
# periodically time out and update the list of handles
# that we are waiting for. It is also critical that we
# send select() a *copy* of the handles list, as we see
# deadlocks when handles are added while select() is
# waiting
ready_handles = select(
list(handles), noop, noop, _poll_interval)[0]
if not ready_handles:
continue
handle = ready_handles[0]
new_data = os.read(handle.read_pipe, io.DEFAULT_BUFFER_SIZE)
if not new_data:
handle.finalize(self.ostreams)
handles.remove(handle)
continue
handle.decoder_buffer += new_data
# At this point, we have new data sitting in the
# handle.decoder_buffer
handle.decodeIncomingBuffer()
# Now, output whatever we have decoded to the output streams
handle.writeOutputBuffer(self.ostreams)
def _transact(self, query):
try:
h = CreateFile(
self.pipe_name, GENERIC_READ|GENERIC_WRITE, 0, None, OPEN_EXISTING, 0, None
)
packet = self._construct_query(query)
WriteFile(h, packet)
read = 0
err, out = ReadFile(h, DI_PIPE_BUF_SIZE)
length = struct.unpack('i', out[:SIZEOF_INT])[0]
while len(out[SIZEOF_INT:]) < length:
out += ReadFile(h, DI_PIPE_BUF_SIZE)[1]
data = json.loads(out[SIZEOF_INT:], encoding='utf-8')
except pywintypes.error as e:
print e
else:
return data
def close(self):
if self._closed:
return False
self._closed = True
# Ensure _conin_pipe empty
if self._process_handle:
try:
TerminateProcess(self._process_handle, 0)
except:
pass
while True:
try:
error, data = ReadFile(self._conout_pipe, 8192)
if not data:
break
except:
break
if self._conerr_pipe:
while True:
try:
error, data = ReadFile(self._conerr_pipe, 8192)
if not data:
break
except:
break
CloseHandle(self._conin_pipe)
CloseHandle(self._conout_pipe)
if self._conerr_pipe:
CloseHandle(self._conerr_pipe)
winpty_free(self._pty)
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception), why:
if why[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
def _recv(self, which, maxsize = 32768):
conn = getattr(self, which)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception), why:
if why[0] in (109, ESHUTDOWN):
return self._close(which)
print "Error: Shell no stdout"
def _child_reader(self, handle):
"""INTERNAL: Reader thread that reads stdout/stderr of the child
process."""
status = 'data'
while True:
try:
err, data = ReadFile(handle, self.maxread)
assert err == 0 # not expecting error w/o overlapped io
except WindowsError as e:
if e.winerror == ERROR_BROKEN_PIPE:
status = 'eof'
data = ''
else:
status = 'error'
data = e.winerror
self.child_output.put((handle, status, data))
if status != 'data':
break
def __recv(logger, conn, maxsize = 1024*512): try: x = msvcrt.get_osfhandle(conn.fileno()) (read, nAvail, nMessage) = PeekNamedPipe(x, 0) if maxsize < nAvail: nAvail = maxsize if nAvail > 0: (errCode, read) = ReadFile(x, nAvail, None) #read = subprocess.Popen._translate_newlines(read) except ValueError: logger.error( "get one unknow error when receive data" ) return '', "unknown error" except (subprocess.pywintypes.error, Exception), why: if why[0] in (109, errno.ESHUTDOWN): if why[0] != 109: logger.error( "get error when read info from pipe %s, pipe id:%d, reason:%s"%(name, which.pid, str(why)) ) return '', str(why[-1]) return '', str(why[-1])
def recv(self, size):
self._logger.debug('recv: %d', size)
chunks = []
while size:
(err, read) = ReadFile(self.fd, size, None)
if err:
raise OSError(err)
chunks.append(read)
size -= len(read)
received = 0
while received < 1:
data = self.r_terminator.recv(1)
received += len(data)
self._logger.debug('recv: read 1 byte from socket, returning data')
return b"".join(chunks)
def pipe_read( self, _pipe, _minimum_to_read ):
## Hackaround since Windows doesn't support select() except for sockets.
dbg_print( 'pipe_read: minimum to read is ' + str( _minimum_to_read ) )
dbg_print( 'pipe_read: sleeping for ' + str( self.delay ) + ' seconds' )
time.sleep( self.delay )
data = ''
osfh = msvcrt.get_osfhandle( _pipe )
( read, count, msg ) = PeekNamedPipe( osfh, 0 )
dbg_print( 'pipe_read: initial count via PeekNamedPipe is ' + str( count ) )
while ( count > 0 ):
dbg_print( 'pipe_read: reading from pipe' )
dbg_print( '' )
( err, tmp ) = ReadFile( osfh, count, None )
data += tmp
dbg_print( 'pipe_read: read ' + str( tmp ) )
( read, count, msg ) = PeekNamedPipe( osfh, 0 )
## Be sure to break the read, if asked to do so,
## after we've read in a line termination.
if _minimum_to_read != 0 and len( data ) > 0 and data[ len( data ) -1 ] == '\n':
if len( data ) >= _minimum_to_read:
dbg_print( 'pipe_read: found termination and read at least the minimum asked for' )
break
else:
dbg_print( 'pipe_read: not all of the data has been read' )
dbg_print( 'pipe_read: returning' )
return data
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno()) # @UndefinedVariable
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except:
return None
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except (ValueError, NameError):
return self._close(which)
except (subprocess.pywintypes.error, Exception) as ex:
if ex[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, _) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(_, read) = ReadFile(x, nAvail, None)
except (ValueError, NameError):
return self._close(which)
except Exception as ex:
if getattr(ex, "args",
None) and ex.args[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _mergedReader(self):
noop = []
handles = self._active_handles
_poll = _poll_interval
_fast_poll_ct = _poll_rampup
new_data = '' # something not None
while handles:
if new_data is None:
# For performance reasons, we use very aggressive
# polling at the beginning (_poll_interval) and then
# ramp up to a much more modest polling interval
# (_poll_rampup_limit) as the process runs and the
# frequency of new data appearing on the pipe slows
if _fast_poll_ct:
_fast_poll_ct -= 1
if not _fast_poll_ct:
_poll *= 10
if _poll < _poll_rampup_limit:
# reset the counter (to potentially increase
# the polling interval again)
_fast_poll_ct = _poll_rampup
else:
new_data = None
if _mswindows:
for handle in list(handles):
try:
pipe = get_osfhandle(handle.read_pipe)
numAvail = PeekNamedPipe(pipe, 0)[1]
if numAvail:
result, new_data = ReadFile(pipe, numAvail, None)
handle.decoder_buffer += new_data
break
except:
handles.remove(handle)
new_data = None
if new_data is None:
# PeekNamedPipe is non-blocking; to avoid swamping
# the core, sleep for a "short" amount of time
time.sleep(_poll)
continue
else:
# Because we could be *adding* handles to the TeeStream
# while the _mergedReader is running, we want to
# periodically time out and update the list of handles
# that we are waiting for. It is also critical that we
# send select() a *copy* of the handles list, as we see
# deadlocks when handles are added while select() is
# waiting
ready_handles = select(list(handles), noop, noop, _poll)[0]
if not ready_handles:
new_data = None
continue
handle = ready_handles[0]
new_data = os.read(handle.read_pipe, io.DEFAULT_BUFFER_SIZE)
if not new_data:
handles.remove(handle)
continue
handle.decoder_buffer += new_data
# At this point, we have new data sitting in the
# handle.decoder_buffer
handle.decodeIncomingBuffer()
# Now, output whatever we have decoded to the output streams
handle.writeOutputBuffer(self.ostreams)
def osnmread(fd, length):
# ReadFile(handle, buf, 1024, &bytes_read, NULL)
# http://docs.activestate.com/activepython/2.7/pywin32/win32file__ReadFile_meth.html
# (int, string) = ReadFile(hFile, buffer/bufSize , ol )
(lread, data) = ReadFile(fd, length, None)
return data
def _merged_reader(*args):
class StreamData(object):
__slots__ = ('read', 'output', 'unbuffer', 'buf', 'data')
def __init__(self, *args):
if _mswindows:
self.read = get_osfhandle(args[1])
else:
self.read = args[1]
self.unbuffer = args[0]
self.output = tuple(x for x in args[2:] if x is not None)
self.buf = ""
self.data = None
def write(self, x):
success = True
for s in self.output:
try:
s.write(x)
except ValueError:
success = False
return success
def flush(self):
for s in self.output:
try:
s.flush()
except ValueError:
pass
raw_stderr = sys.__stderr__
if raw_stderr is None:
# There are cases, e.g., in Anaconda, where there is no stdout
# for the original process because, for example, it was started
# in a windowing environment.
raw_stderr = sys.stderr
try:
encoding = stream.encoding
except:
encoding = None
if encoding is None:
try:
encoding = raw_stderr.encoding
except:
pass
if encoding is None:
encoding = 'utf-8'
streams = {}
for s in args:
tmp = StreamData(*s)
streams[tmp.read] = tmp
handles = sorted(streams.keys(), key=lambda x: -1 * streams[x].unbuffer)
noop = []
while handles:
if _mswindows:
new_data = None
for h in handles:
try:
numAvail = PeekNamedPipe(h, 0)[1]
if numAvail == 0:
continue
result, new_data = ReadFile(h, 1, None)
except:
handles.remove(h)
new_data = None
if new_data is None:
continue
else:
h = select(handles, noop, noop)[0]
if not h:
break
h = h[0]
new_data = os.read(h, 1)
if not new_data:
handles.remove(h)
continue
s = streams[h]
if s.data is None:
s.data = new_data
else:
s.data += new_data
char = s.data.decode(encoding)
if char.encode(encoding) != s.data:
continue
s.data = None
if s.unbuffer:
writeOK = s.write(char)
s.buf += char
if char[-1] != "\n":
continue
if s.unbuffer:
s.flush()
else:
writeOK = s.write(s.buf)
if writeOK:
s.buf = ""
writeOK = True
for s in itervalues(streams):
if s.buf:
writeOK &= s.write(s.buf)
if s.data:
writeOK &= s.write(s.data.decode(encoding))
s.flush()
if not writeOK and raw_stderr is not None:
raw_stderr.write("""
ERROR: pyutilib.subprocess: output stream closed before all subprocess output
was written to it. The following was left in the subprocess buffer:
'%s'
""" % (buf, ))
if data:
raw_stderr.write(
"""The following undecoded unicode output was also present:
'%s'
""" % (data, ))