def poll(self, timeout=-1):
"""Overridden method to ensure that the green version of
Poller is used.
Behaves the same as :meth:`zmq.core.Poller.poll`
"""
if timeout is None:
timeout = -1
if timeout < 0:
timeout = -1
rlist = None
wlist = None
xlist = None
if timeout > 0:
tout = gevent.Timeout.start_new(timeout/1000.0)
try:
# Loop until timeout or events available
rlist, wlist, xlist = self._get_descriptors()
while True:
events = super(GreenPoller, self).poll(0)
if events or timeout == 0:
return events
# wait for activity on sockets in a green way
select.select(rlist, wlist, xlist)
except gevent.Timeout, t:
if t is not tout:
raise
return []
def wait(self, timeout):
r, w = os.pipe()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
def dothat():
"""
Wait for I/O operation
"""
print('Start in dothat at {}'.format(tic()))
select.select([], [], [], 2)
print('In dothat again at {}'.format(tic()))
def _read_forward_sock(self, forward_sock, channel):
while True:
if channel.eof():
logger.debug("Channel closed")
return
try:
data = forward_sock.recv(1024)
except Exception:
logger.exception("Forward socket read error:")
sleep(1)
continue
data_len = len(data)
if data_len == 0:
continue
data_written = 0
while data_written < data_len:
try:
rc, bytes_written = channel.write(data[data_written:])
except Exception:
logger.exception("Channel write error:")
sleep(1)
continue
data_written += bytes_written
if rc == LIBSSH2_ERROR_EAGAIN:
select((), ((self.client.sock, )), (), timeout=0.001)
def gr2():
# Busy waits for a second, but we don't want to stick around...
print('Started Polling: %s' % tic())
# gevent.select.select(rlist, wlist, xlist, timeout=None)
select.select([], [], [], 2)
print('Ended Polling: %s' % tic())
def gr2():
# Busy waits for a second, but we don't want to stick around...
print('Started Polling: %s' % tic())
# gevent.select.select(rlist, wlist, xlist, timeout=None)
select.select([], [], [], 2)
print('Ended Polling: %s' % tic())
def gr1():
# Busy waits for a second, but we don't want to stick around...
print('Started Polling: %s' % tic())
select.select([], [], [], 3) # Tim: Block this greenlet until the events are ready,
# but no events are specified so block forever,
# or until the timeout of 3 seconds expires.
print('Ended Polling: %s' % tic())
def wait(self, timeout):
r, w = os.pipe()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
def poll(self, timeout=-1):
"""Overridden method to ensure that the green version of
Poller is used.
Behaves the same as :meth:`zmq.core.Poller.poll`
"""
if timeout is None:
timeout = -1
if timeout < 0:
timeout = -1
rlist = None
wlist = None
xlist = None
if timeout > 0:
tout = gevent.Timeout.start_new(timeout / 1000.0)
try:
# Loop until timeout or events available
rlist, wlist, xlist = self._get_descriptors()
while True:
events = super(GreenPoller, self).poll(0)
if events or timeout == 0:
return events
# wait for activity on sockets in a green way
select.select(rlist, wlist, xlist)
except gevent.Timeout, t:
if t is not tout:
raise
return []
def test_long(self):
sock = socket.socket()
try:
select.select(
[six.builtins.long(sock.fileno())], [], [], 0.001)
finally:
sock.close()
def _read_forward_sock(self, forward_sock, channel):
while True:
if channel.eof():
logger.debug("Channel closed")
return
try:
data = forward_sock.recv(1024)
except Exception:
logger.exception("Forward socket read error:")
sleep(1)
continue
data_len = len(data)
if data_len == 0:
continue
data_written = 0
while data_written < data_len:
try:
rc, bytes_written = channel.write(data[data_written:])
except Exception:
logger.exception("Channel write error:")
sleep(1)
continue
data_written += bytes_written
if rc == LIBSSH2_ERROR_EAGAIN:
select((), ((self.client.sock,)), (), timeout=0.001)
def _read_channel(self, forward_sock, channel):
while True:
if channel.eof():
logger.debug("Channel closed")
return
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(1)
continue
while size == LIBSSH2_ERROR_EAGAIN or size > 0:
if size == LIBSSH2_ERROR_EAGAIN:
select((self.client.sock,), (), (), timeout=0.001)
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(1)
continue
while size > 0:
try:
forward_sock.sendall(data)
except Exception as ex:
logger.error(
"Error sending data to forward socket - %s", ex)
sleep(.5)
continue
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(.5)
def _read_channel(self, forward_sock, channel):
while True:
if channel.eof():
logger.debug("Channel closed")
return
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(1)
continue
while size == LIBSSH2_ERROR_EAGAIN or size > 0:
if size == LIBSSH2_ERROR_EAGAIN:
select((self.client.sock, ), (), (), timeout=0.001)
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(1)
continue
while size > 0:
try:
forward_sock.sendall(data)
except Exception as ex:
logger.error(
"Error sending data to forward socket - %s", ex)
sleep(.5)
continue
try:
size, data = channel.read()
except Exception as ex:
logger.error("Error reading from channel - %s", ex)
sleep(.5)
def test_long(self):
sock = socket.socket()
try:
select.select(
[six.builtins.long(sock.fileno())], [], [], 0.001)
finally:
sock.close()
def read(self):
while True:
try:
return super(inotify, self).read()
except OSError as exc:
if exc.errno != errno.EAGAIN:
raise
select([self], [], [])
def read(self):
while True:
try:
return super(inotify, self).read()
except OSError as exc:
if exc.errno != errno.EAGAIN:
raise
select([self], [], [])
def _open_channel(self, fw_host, fw_port, local_port):
channel = self.session.direct_tcpip_ex(fw_host, fw_port, '127.0.0.1',
local_port)
while channel == LIBSSH2_ERROR_EAGAIN:
select((self.client.sock, ), (self.client.sock, ), ())
channel = self.session.direct_tcpip_ex(fw_host, fw_port,
'127.0.0.1', local_port)
return channel
def select_greenlet_runner(fd, event):
"""Sets event when data becomes available to read on fd."""
while True:
event.set()
try:
select([fd], [], [])[0]
except ValueError:
break
def select_greenlet_runner(fd, event):
"""Sets event when data becomes available to read on fd."""
while True:
event.set()
try:
select([fd], [], [])[0]
except ValueError:
break
def bulk_operation(self, serializer, index, client, **options):
procs = options.get('multi')
if not procs:
procs = os.cpu_count()
procs = procs if procs > 1 else 2
data_size = serializer.fetch_data_length()
chunk_size = int(data_size / (procs * 4))
def parallel_params():
for chunk_num in range(procs * 4 + 1):
_options = options.copy()
_options['parallel_chunk_num'] = chunk_num
_options['parallel_chunk_size'] = chunk_size
yield hash(serializer), index, _options
if not self.pipe_fd_mapping:
# reset multiprocessing.Process-sensitive elements just before forking
self.parallel_prep()
for n in range(procs):
h1, h2 = gipc.pipe(duplex=True)
self.pipe_fd_mapping[h1._reader._fd] = (
h1,
gipc.start_process(process_func,
args=(h2, ),
name="esdocs-proc-{}".format(n)))
# wait until sub-process workers check in
ready = 0
while ready != procs:
readable, _, _ = select(self.pipe_fd_mapping.keys(), [], [])
for fd in readable:
h1, proc = self.pipe_fd_mapping[fd]
msg, data = h1.get()
if msg == 'GOOD-MORNING-DAVE':
ready += 1
parallel_chunk_params = parallel_params()
# populate all the processes with an even number of chunks to process
chunks_remaining = True
chunks_to_process = 0
while chunks_remaining:
for h1, proc in self.pipe_fd_mapping.values():
try:
h1.put(next(parallel_chunk_params))
chunks_to_process += 1
except StopIteration:
chunks_remaining = False
break
while chunks_to_process > 0:
readable, _, _ = select(self.pipe_fd_mapping.keys(), [], [])
for fd in readable:
h1, proc = self.pipe_fd_mapping[fd]
msg, data = h1.get()
if msg == 'DONE-CHUNK':
chunks_to_process -= 1
def _open_channel(self, fw_host, fw_port, local_port):
channel = self.session.direct_tcpip_ex(
fw_host, fw_port, '127.0.0.1',
local_port)
while channel == LIBSSH2_ERROR_EAGAIN:
select((self.client.sock,), (self.client.sock,), ())
channel = self.session.direct_tcpip_ex(
fw_host, fw_port, '127.0.0.1',
local_port)
return channel
def _handle_socket_read(handler, sock):
collected = b''
while True:
# a full packet before calling on_packet in the handler class
# Ideally instead of 10 msec timeout, this should have been
# infinity (that is None), but post that change the messaging
# stops working because it is possible that this would
# block other actions, like handling messages on the queues.
# TODO: find a better solution.
# Additionally, anything lower than 10 msec would unnecessarily consume
# CPU, because epoll_wait() will have a time out 1 msec if the timeout
# here is set to 0.0, which is bad for idle python nodes.
# Note that setting a value of 0.01 or 10 msec would imply that
# python node would not be able to send replies faster than 10 msec
# to Erlang node, but this is an acceptable delay condering the
# idle python node cpu consumption issue mentioned above.
# update: set lower timeout of 1 millisecond but do a select for
# longer time 1 second further down below to save cpu cycles
# when there are no messages.
# This is a HACK
ready = select.select([sock], [], [], 0.001)
try:
if ready[0]:
data = sock.recv(4096)
# print("data in: %s" % hex_bytes(data))
collected += data
# Try and consume repeatedly if multiple messages arrived
# in the same packet
while True:
collected1 = handler.consume(collected)
if collected1 is None:
print("Protocol requested to disconnect the socket")
sock.close()
handler.on_connection_lost()
return
if collected1 == collected:
break # could not consume any more
collected = collected1
else:
handler.handle_inbox()
# HACK to keep idle CPU down to 0.3% while
# trying to maintain lower latency
select.select([sock], [], [], 1.0)
except select.error:
# Disconnected probably or another error
break
sock.close()
handler.on_connection_lost()
def test_wait_channel_error(self):
DNSResolver._channel = channel = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(select, 'select')
channel.getsock().AndReturn(('read', 'write'))
channel.timeout().AndReturn(1.0)
select.select('read', 'write', [], 1.0).AndRaise(ValueError(13))
channel.cancel()
self.mox.ReplayAll()
with self.assertRaises(ValueError):
DNSResolver._wait_channel()
self.assertIsNone(DNSResolver._channel)
def test_wait_channel_error(self):
DNSResolver._channel = channel = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(select, 'select')
channel.getsock().AndReturn(('read', 'write'))
channel.timeout().AndReturn(1.0)
select.select('read', 'write', [], 1.0).AndRaise(ValueError(13))
channel.cancel()
self.mox.ReplayAll()
with self.assertRaises(ValueError):
DNSResolver._wait_channel()
self.assertIsNone(DNSResolver._channel)
def inputhook_gevent():
"""PyOS_InputHook python hook for Gevent.
"""
allow_CTRL_C()
try:
select.select([sys.stdin], [], [])
except:
from traceback import print_exc
print_exc()
return 0
def _listen(websocket_fd):
"""
This will listen to the websocket file descriptor in a gevent-friendly way, and notify when the fd is ready
:param websocket_fd: the websocket file descriptor
:return:
"""
while True:
# select fd with a 3s timeout, so that we can ping client from time to time
select([websocket_fd], [], [], timeout=3)
_websocket_recv_event.set()
def pithos_xseg_wait_signal_green(ctx, sd, timeout):
posixfd_sd = cast(sd, POINTER(posixfd_signal_desc))
fd = posixfd_sd.contents.fd
select.select([fd], [], [], timeout / 1000000.0)
while True:
try:
os.read(fd, 512)
except OSError as (e, msg):
if e == 11:
break
else:
raise OSError(e, msg)
def pithos_xseg_wait_signal_green(ctx, sd, timeout):
posixfd_sd = cast(sd, POINTER(posixfd_signal_desc))
fd = posixfd_sd.contents.fd
select.select([fd], [], [], timeout / 1000000.0)
while True:
try:
os.read(fd, 512)
except OSError as (e, msg):
if e == 11:
break
else:
raise OSError(e, msg)
def gevent_wait_callback(conn, timeout=None):
"""A wait callback useful to allow gevent to work with Psycopg."""
while 1:
state = conn.poll()
if state == extensions.POLL_OK:
break
elif state == extensions.POLL_READ:
select([conn.fileno()], [], [], timeout=timeout)
elif state == extensions.POLL_WRITE:
select([], [conn.fileno()], [], timeout=timeout)
else:
raise OperationalError("Bad result from poll: %r" % state)
def handle_read(self):
rlist = (self._socket,)
while True:
try:
select.select(rlist, (), ())
except Exception as err:
return
try:
buf = self._socket.recv(self.in_buffer_size)
except socket.error as err:
if not is_timeout(err):
self.defunct(err)
return # leave the read loop
if buf:
self._iobuf.write(buf)
while True:
pos = self._iobuf.tell()
if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
# we don't have a complete header yet or we
# already saw a header, but we don't have a
# complete message yet
break
else:
# have enough for header, read body len from header
self._iobuf.seek(4)
body_len_bytes = self._iobuf.read(4)
body_len = int32_unpack(body_len_bytes)
# seek to end to get length of current buffer
self._iobuf.seek(0, os.SEEK_END)
pos = self._iobuf.tell()
if pos - 8 >= body_len:
# read message header and body
self._iobuf.seek(0)
msg = self._iobuf.read(8 + body_len)
# leave leftover in current buffer
leftover = self._iobuf.read()
self._iobuf = StringIO()
self._iobuf.write(leftover)
self._total_reqd_bytes = 0
self.process_msg(msg, body_len)
else:
self._total_reqd_bytes = body_len + 8
break
else:
log.debug("connection closed by server")
self.close()
def gevent_wait_callback(conn, timeout=None):
"""A wait callback useful to allow gevent to work with Psycopg."""
while 1:
state = conn.poll()
if state == extensions.POLL_OK:
break
elif state == extensions.POLL_READ:
select([conn.fileno()], [], [], timeout=timeout)
elif state == extensions.POLL_WRITE:
select([], [conn.fileno()], [], timeout=timeout)
else:
raise OperationalError("Bad result from poll: %r" % state)
def run(self):
#Fire off some greenlits to handing reading and writing
try:
print("Starting Read/Write Loops")
tasks = [gevent.spawn(raise_exceptions(self._read)),
gevent.spawn(raise_exceptions(self._write))]
#Wait for a socket exception and raise the flag
select.select([], [], [self._socket]) # Yield
raise self.SocketError('Socket Exception')
finally: # Make sure we kill the tasks
print("Killing read and write loops")
gevent.killall(tasks)
def handle_read(self):
rlist = (self._socket,)
while True:
try:
select.select(rlist, (), ())
except Exception as err:
return
try:
buf = self._socket.recv(self.in_buffer_size)
except socket.error as err:
if not is_timeout(err):
self.defunct(err)
return # leave the read loop
if buf:
self._iobuf.write(buf)
while True:
pos = self._iobuf.tell()
if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
# we don't have a complete header yet or we
# already saw a header, but we don't have a
# complete message yet
break
else:
# have enough for header, read body len from header
self._iobuf.seek(4)
body_len_bytes = self._iobuf.read(4)
body_len = int32_unpack(body_len_bytes)
# seek to end to get length of current buffer
self._iobuf.seek(0, os.SEEK_END)
pos = self._iobuf.tell()
if pos - 8 >= body_len:
# read message header and body
self._iobuf.seek(0)
msg = self._iobuf.read(8 + body_len)
# leave leftover in current buffer
leftover = self._iobuf.read()
self._iobuf = StringIO()
self._iobuf.write(leftover)
self._total_reqd_bytes = 0
self.process_msg(msg, body_len)
else:
self._total_reqd_bytes = body_len + 8
break
else:
log.debug("connection closed by server")
self.close()
def run(self):
"""Watch for notifications.
Obtain a new connection to the PostgreSQL database, attach to
all the channels needed to fulfill the requirements posed by
the callbacks we have to execute, notify the service by calling
the initialization callback and then start waiting for events.
When an event arrives parse it (both the channel name and the
payload), check which callbacks it triggers and fire them.
"""
while True:
try:
# Obtain a connection.
conn = custom_psycopg2_connection()
conn.autocommit = True
# Execute all needed LISTEN queries.
curs = conn.cursor()
for event, table_name in self._channels:
curs.execute(b"LISTEN {0}_{1};".format(event, table_name))
# Notify the service that we're ready to go: we're attached
# to all notification channels. It can start fetching its
# objects without fearing that we'll miss any update to them.
for callback in self._init_callbacks:
gevent.spawn(callback)
# Listen.
while True:
# FIXME Use a timeout?
select.select([conn], [], [])
conn.poll()
for notify in conn.notifies:
# Parse the notification.
event, _, table_name = notify.channel.partition(b'_')
rows = notify.payload.split(b'\n')
pkey = tuple(int(i) for i in rows[0].split(b' '))
cols = set(rows[1:])
for item in self._callbacks:
if item[1] == event and item[2] == table_name and \
(len(item[3]) == 0 or
not item[3].isdisjoint(cols) > 0):
gevent.spawn(item[0], *pkey)
del conn.notifies[:]
except psycopg2.OperationalError:
logger.warning("Lost connection with database.")
gevent.sleep(1)
def _wait_select(libssh2_sess, sock):
'''
Find out from libssh2 if its blocked on read or write and wait accordingly
Return immediately if libssh2 is not blocked
'''
blockdir = libssh2_sess.blockdirections()
if blockdir == 0:
return
readfds = [sock] if (blockdir & 01) else []
writefds = [sock] if (blockdir & 02) else []
select(readfds, writefds, [])
return
def read_input(self):
while True:
try:
select.select([sys.stdin], [], [])
msg_text = raw_input()
msg = Message(instance = -1,
id = self._id,
msg = msg_text,
type = Message.PROPOSAL)
self.send(msg, 'proposers')
except EOFError, KeyboardInterrupt:
exit(0)
def _relay_command(self, message, transform=lambda x: x):
if self._command_bus is None:
return
# Send the command.
wfd = self._command_bus.getsockopt(nnpy.SOL_SOCKET, nnpy.SNDFD)
rfd = self._command_bus.getsockopt(nnpy.SOL_SOCKET, nnpy.RCVFD)
rl, wl, xl = select.select([], [wfd], [])
self._command_bus.send(message)
# Wait for a response and dispatch it via the web socket.
rl, wl, xl = select.select([rfd], [], [])
data = transform(self._command_bus.recv())
self._socket.send('command@' + data)
def poll(self, timeout=-1):
"""Overridden method to ensure that the green version of
Poller is used.
Behaves the same as :meth:`zmq.core.Poller.poll`
"""
if timeout is None:
timeout = -1
if timeout < 0:
timeout = -1
rlist = None
wlist = None
xlist = None
if timeout > 0:
tout = gevent.Timeout.start_new(timeout / 1000.0)
else:
tout = None
try:
# Loop until timeout or events available
rlist, wlist, xlist = self._get_descriptors()
while True:
events = super(_Poller, self).poll(0)
if events or timeout == 0:
return events
# wait for activity on sockets in a green way
# set a minimum poll frequency,
# because gevent < 1.0 cannot be trusted to catch edge-triggered FD events
_bug_timeout = gevent.Timeout.start_new(
self._gevent_bug_timeout)
try:
select.select(rlist, wlist, xlist)
except gevent.Timeout as t:
if t is not _bug_timeout:
raise
finally:
_bug_timeout.cancel()
except gevent.Timeout as t:
if t is not tout:
raise
return []
finally:
if timeout > 0:
tout.cancel()
def run(self):
#Fire off some greenlits to handing reading and writing
try:
print("Starting Read/Write Loops")
tasks = [
gevent.spawn(raise_exceptions(self._read)),
gevent.spawn(raise_exceptions(self._write))
]
#Wait for a socket exception and raise the flag
select.select([], [], [self._socket]) # Yield
raise self.SocketError('Socket Exception')
finally: # Make sure we kill the tasks
print("Killing read and write loops")
gevent.killall(tasks)
def _relay_command(self, message, transform=lambda x: x):
if self._command_bus is None:
return
# Send the command.
wfd = self._command_bus.getsockopt(nnpy.SOL_SOCKET, nnpy.SNDFD)
rfd = self._command_bus.getsockopt(nnpy.SOL_SOCKET, nnpy.RCVFD)
rl, wl, xl = select.select([], [wfd], [])
self._command_bus.send(message)
# Wait for a response and dispatch it via the web socket.
rl, wl, xl = select.select([rfd], [], [])
data = transform(self._command_bus.recv())
self._socket.send('command@' + data)
def receive(self):
ping_timeout = timedelta(seconds=self.activity_timeout)
connection_timeout = timedelta(seconds=ping_timeout.seconds * 2)
ping_sent = False
last_heartbeat = datetime.now()
while True:
try:
if datetime.now() - last_heartbeat > ping_timeout and \
not ping_sent:
self.queue.put(BusinessObject({'event': 'ping'}, None))
ping_sent = True
elif datetime.now() - last_heartbeat > connection_timeout and \
ping_sent is True:
raise ConnectionTimeout(
"Timeout! Last object received at %s (is the server responding to ping?)"
% last_heartbeat)
if not self.queue.empty():
rlist, wlist, xlist = select.select([self.socket],
[self.socket], [], 0.2)
if len(wlist) > 0:
obj = self.queue.get()
obj.serialize(socket=self.socket)
else:
rlist, wlist, xlist = select.select([self.socket], [], [],
0.2)
if len(rlist) > 0:
obj = BusinessObject.read_from_socket(self.socket)
if obj is None:
raise InvalidObject
last_heartbeat = datetime.now()
ping_sent = False
if obj.event == 'services/discovery':
response = self.handle_discovery(obj)
if response is not None:
self.queue.put(response)
elif self.should_handle(obj):
response = self.handle(obj)
if response is not None:
self.queue.put(response)
# TODO: implement functionality for other than pure stimuli induced behavior
except InvalidObject, ivo:
self.socket.close()
break
except KeyboardInterrupt, kbi:
self.socket.close()
raise kbi
def poll(self, timeout=-1):
"""Overridden method to ensure that the green version of
Poller is used.
Behaves the same as :meth:`zmq.core.Poller.poll`
"""
if timeout is None:
timeout = -1
if timeout < 0:
timeout = -1
rlist = None
wlist = None
xlist = None
if timeout > 0:
tout = gevent.Timeout.start_new(timeout / 1000.0)
else:
tout = None
try:
# Loop until timeout or events available
rlist, wlist, xlist = self._get_descriptors()
while True:
events = super(_Poller, self).poll(0)
if events or timeout == 0:
return events
# wait for activity on sockets in a green way
# set a minimum poll frequency,
# because gevent < 1.0 cannot be trusted to catch edge-triggered FD events
_bug_timeout = gevent.Timeout.start_new(self._gevent_bug_timeout)
try:
select.select(rlist, wlist, xlist)
except gevent.Timeout as t:
if t is not _bug_timeout:
raise
finally:
_bug_timeout.cancel()
except gevent.Timeout as t:
if t is not tout:
raise
return []
finally:
if timeout > 0:
tout.cancel()
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
def socket_handler(self):
log("Reactor thread started")
self.connect()
while True:
select([self.central.stack], [], [])
event = self.central.stack.handle_data()
if event.type == BTEvent.SCAN_DATA:
addr, type, data = event.data
log("Saw %s (%s)" %
(addr, "public" if type == 0 else "random"))
try:
if len(data) > len(SEEN[addr][1]):
SEEN[addr] = (type, data)
dump_gap(data)
except KeyError:
SEEN[addr] = (type, data)
dump_gap(data)
if addr.lower() == self.target.lower(
) and self.connected == False:
pass
elif event.type == BTEvent.CONNECTED:
self.connected = True
log("Connected!")
self.onconnect()
elif event.type == BTEvent.DISCONNECTED:
self.connected = False
log("Disconnected")
log("Attempting reconnection")
self.connect()
#self.reset()
# log("Reset")
elif event.type == BTEvent.ATT_DATA:
pkt = event.data
# ack handle value notification
if pkt.opcode == 0x1d:
self.central.stack.raw_att("\x1e")
handle = pkt.payload.load[0]
null = pkt.payload.load[1]
body = pkt.payload.load[2:]
# log_in("%x : %s" % (ord(handle), repr(body)))
mklog("< %x" % ord(handle))(repr(body))
self.buffer.push(body)
elif pkt.opcode == 0x13:
QUEUE.put(1)
elif event.type != BTEvent.NONE:
log(repr(event))
def handle(self, server, client, client_address):
# Simple select bag with client and server sockets
r, __, __ = select([client, server], [], [], 0.5)
if client in r:
self.recv_send(client, server, request=True)
if server in r:
self.recv_send(server, client, request=False)
def meeks_read_from_socks_thread(self):
while not self.finish.is_set():
try:
readable, _, _ = select.select(self.allsocks, [], [], CLIENT_MAX_POLL_INTERVAL)
if not readable:
self.timer.count(CLIENT_MAX_POLL_INTERVAL)
if self.timer.timeout():
break
else:
self.timer.reset()
if self.socksconn in readable:
if self.udpsock:
raise RelaySessionError("unexcepted read-event from tcp socket in UDP session")
data = self.socksconn.recv(MAX_PAYLOAD_LENGTH)
if not data:
raise RelaySessionError("peer closed")
self.out_queue.put(data)
continue
if self.udpsock and self.udpsock in readable:
data, _ = self.udpsock.recvfrom(MAX_PAYLOAD_LENGTH)
if data:
self.out_queue.put(data)
except Exception as ex:
log.error("[Exception][meeks_read_from_socks_thread] %s:%s" % (self.sessionid, str(ex)))
break
self.finish.set()
def listen( self, unmarshaller = pipe_colon_unmarshall ):
"""
Subscribe to the channel and send notification payloads to the
results Queue.
:param function unmarshaller: Function to pass the `notify.payload` string into for unmarshalling into python object (ie. dict) data
"""
self.stop_event = stop_event = gevent.event.Event()
self.cur = self.conn.cursor()
self.cur.execute( "LISTEN %s;" % ( self.channel_name, ) )
while 1:
if self.stop_event.is_set():
return
# TODO: test select with timeout and try/except
# maybe try to listen on all channels with just one connection with short timeouts on the select
# and then cycle through...
if select( [ self.conn ], [], [] ) == ( [], [], [] ):
print "LISTEN timeout."
else:
if self.stop_event.is_set():
return
self.conn.poll()
while self.conn.notifies:
if self.stop_event.is_set():
return
notify = self.conn.notifies.pop()
payload_data = unmarshaller( notify.payload )
for q_id in self.subscribers.iterkeys():
self.subscribers[ q_id ].put( payload_data )
def pipe_udp(tcpsocks, csock, rsock, ctimeout, rtimeout, caddrchecker, c2r,
r2c):
rlist = tcpsocks + [csock, rsock]
csock_timer = 0
rsock_timer = 0
while True:
readable, _, _ = select.select(rlist, [], [], timeout=1)
if not readable:
csock_timer += 1
rsock_timer += 1
if csock_timer > ctimeout or rsock_timer > rtimeout:
return
else:
continue
for s in tcpsocks:
if s in readable:
return
if csock in readable:
csock_timer = 0
fromdata, fromaddr = csock.recvfrom(65536)
if caddrchecker(fromaddr[0], fromaddr[1]):
todata, toaddr = c2r(fromdata, fromaddr)
if todata and toaddr:
rsock.sendto(todata, toaddr)
if rsock in readable:
rsock_timer = 0
fromdata, fromaddr = rsock.recvfrom(65536)
todata, toaddr = r2c(fromdata, fromaddr)
if todata and toaddr:
csock.sendto(todata, toaddr)
def handle(self, server, client, client_address):
# Simple select bag with client and server sockets
r, __, __ = select([client, server], [], [], 0.5)
if client in r:
self.recv_send(client, server, request=True)
if server in r:
self.recv_send(server, client, request=False)
def _run(self):
self.r_sock.settimeout(3)
try:
self.r_sock.connect(self.remote)
except:
return self.sock.close()
while True:
wset = []
if self.writes:
wset.append(self.sock)
if self.r_writes:
wset.append(self.r_sock)
ifds, wfds, efds = select([self.sock, self.r_sock],
wset, [self.sock, self.r_sock])
try:
self.on_read(ifds)
self.on_write(wfds)
except LinkBroken:
self.close()
break
if efds:
self.close()
break
def listen(self, unmarshaller=pipe_colon_unmarshall):
"""
Subscribe to the channel and send notification payloads to the
results Queue.
:param function unmarshaller: Function to pass the `notify.payload` string into for unmarshalling into python object (ie. dict) data
"""
self.stop_event = stop_event = gevent.event.Event()
self.cur = self.conn.cursor()
self.cur.execute("LISTEN %s;" % (self.channel_name, ))
while 1:
if self.stop_event.is_set():
return
# TODO: test select with timeout and try/except
# maybe try to listen on all channels with just one connection with short timeouts on the select
# and then cycle through...
if select([self.conn], [], []) == ([], [], []):
print "LISTEN timeout."
else:
if self.stop_event.is_set():
return
self.conn.poll()
while self.conn.notifies:
if self.stop_event.is_set():
return
notify = self.conn.notifies.pop()
payload_data = unmarshaller(notify.payload)
for q_id in self.subscribers.iterkeys():
self.subscribers[q_id].put(payload_data)
def is_connection_dropped(conn): # Platform-specific
"""
Returns True if the connection is dropped and should be closed.
:param conn:
:class:`httplib.HTTPConnection` object.
Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us.
"""
sock = getattr(conn, 'sock', False)
if sock is False: # Platform-specific: AppEngine
return False
if sock is None: # Connection already closed (such as by httplib).
return True
if not poll:
if not select: # Platform-specific: AppEngine
return False
try:
return select([sock], [], [], 0.0)[0]
except socket.error:
return True
# This version is better on platforms that support it.
p = poll()
p.register(sock, POLLIN)
for (fno, ev) in p.poll(0.0):
if fno == sock.fileno():
# Either data is buffered (bad), or the connection is dropped.
return True
def read_process():
proc = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# pass data until client disconnects, then terminate
# see https://stackoverflow.com/questions/18511119/stop-processing-flask-route-if-request-aborted
try:
awaiting = [proc.stdout, proc.stderr]
while awaiting:
# wait for output on one or more pipes, or for proc to close a pipe
ready, _, _ = select(awaiting, [], [])
for pipe in ready:
line = pipe.readline()
if line:
# some output to report
yield line.rstrip() + b'\n'
else:
# EOF, pipe was closed by proc
awaiting.remove(pipe)
if proc.poll() is None:
print("process closed stdout and stderr but didn't terminate; terminating now.")
proc.terminate()
except GeneratorExit:
# occurs when new output is yielded to a disconnected client
print("client disconnected, killing process")
proc.terminate()
# wait for proc to finish and get return code
ret_code = proc.wait()
print(f"process return code: {ret_code}")
def _read(self):
a = select.select([self.ssl_socket], [self.ssl_socket], [self.ssl_socket],10)
print "I can read from the socket if I choose to", a
try:
self.read_data = self.ssl_socket.recv(1024)
except Exception as e:
self.error = self._error_handler(e)
def handle_batch_client(sock):
recvbuf = ""
while True:
rds, _, _ = select.select([sock], [], [], 60 * 5)
if not rds:
break
data = sock.recv(1024)
if not data:
break
recvbuf += data
pos = recvbuf.find("\r\n\r\n")
if pos == -1:
continue
parser = HttpParser()
nparsed = parser.execute(recvbuf, pos + 4)
if nparsed != pos + 4:
logging.debug("pos:%d, nparsed:%d, recvbuf:%r", pos, nparsed, recvbuf)
assert nparsed == pos + 4
assert parser.is_headers_complete()
headers = parser.get_headers()
content_length = int(headers["Content-Length"]) if headers.has_key("Content-Length") else 0
logging.debug("content length:%d", content_length)
recvbuf = recvbuf[pos + 4 :]
preread = recvbuf[:content_length]
recvbuf = recvbuf[content_length:]
keepalived = handle_request(sock, parser, preread)
if not keepalived:
break
logging.debug("close client")
sock.close()
def ffmpeg_pipe_stream(stream_url):
global PLEX_FFMPEG_PATH, PLEX_BUFFER_SIZE
pipe_cmd = "%s -re -i %s " \
"-codec copy " \
"-nostats " \
"-loglevel 0 " \
"-bsf:v h264_mp4toannexb " \
"-f mpegts " \
"-tune zerolatency " \
"pipe:1" % (PLEX_FFMPEG_PATH, cmd_quote(stream_url))
p = subprocess.Popen(shlex.split(pipe_cmd),
stdout=subprocess.PIPE,
bufsize=-1)
try:
pipes = [p.stdout]
while pipes:
ready, _, _ = select(pipes, [], [])
for pipe in ready:
data = pipe.read(PLEX_BUFFER_SIZE << 10)
if data:
yield data
else:
pipes.remove(pipe)
except Exception:
pass
except GeneratorExit:
pass
try:
p.terminate()
except Exception:
pass
return
def _run(self):
client = self.client
logger.info(u"Receiver handling connection from {0}".format(client.address))
last_activity = datetime.now()
while True:
if client.server and last_activity + timedelta(minutes=30) < datetime.now():
client.close('inactivity')
return
rlist, wlist, xlist = select([client.socket], [], [], timeout=30.0)
if len(rlist) == 1:
# logger.debug(u"Attempting to read an object from {0}".format(self.socket))
try:
obj = BusinessObject.read_from_socket(client.socket)
if obj is None:
client.close("couldn't read object")
return
# logger.debug(u"Successfully read object {0}".format(str(obj)))
logger.debug(u"<< {0}: {1}".format(client, obj))
client.gateway.send(obj, client)
last_activity = datetime.now()
except InvalidObject, ivo:
client.close(u"{0}".format(ivo))
return
except socket.error, e:
client.close(u"{0}".format(e))
return
except IOError, ioe:
client.close(u"{0}".format(e))
return
