def __receive_thread_worker(self):
waitobj = IOWait()
waitobj.watch( self._socket, read=True)
### BMX poller = select.poll()
### BMX poller.register(self._socket, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR)
__buffer = b''
self._is_receiving = True
self._receiving_callback and self._receiving_callback(self)
while self._is_connected and self.__running:
### BMX events = poller.poll(1000)
events = waitobj.wait(1000) ### BMX
### BMX for fd, event in events:
for fileno, read, write in events: ### BMX
### BMX if event & select.POLLHUP:
### BMX self.logger.warning("Client socket closed")
# Check if POLLIN event triggered
### BMX if event & (select.POLLIN | select.POLLPRI):
if read:
msg = self._socket.recv(4096)
# Check if incoming message is not empty
if msg:
# If we transfer in text mode decode message to string
if not self._binary:
msg = str.rstrip(str(msg, 'utf-8'))
# If we work in line mode (with a terminator) slice buffer into single chunks based on terminator
if self.terminator:
__buffer += msg
while True:
# terminator = int means fixed size chunks
if isinstance(self.terminator, int):
i = self.terminator
if i > len(__buffer):
break
# terminator is str or bytes means search for it
else:
i = __buffer.find(self.terminator)
if i == -1:
break
i += len(self.terminator)
line = __buffer[:i]
__buffer = __buffer[i:]
if self._data_received_callback is not None:
self._data_received_callback(self, line)
# If not in terminator mode just forward what we received
else:
if self._data_received_callback is not None:
self._data_received_callback(self, msg)
# If empty peer has closed the connection
else:
# Peer connection closed
self.logger.warning("Connection closed by peer {}".format(self._host))
self._is_connected = False
### BMX poller.unregister()
waitobj.unwatch(self._socket)
self._disconnected_callback and self._disconnected_callback(self)
if self._autoreconnect:
self.logger.debug("Autoreconnect enabled for {}".format(self._host))
self.connect()
self._is_receiving = False
class SocketStatsCollector(BaseStatsCollector):
def __init__(self, streamer, name, callback_time=1., io_loop=None):
super(SocketStatsCollector, self).__init__(streamer, name,
callback_time, io_loop)
# if gevent is installed, we'll use a greenlet,
# otherwise we'll use a thread
try:
import gevent # NOQA
self.greenlet = True
except ImportError:
self.greenlet = False
self._rstats = defaultdict(int)
if self.greenlet:
self.poller = SelectIOWait()
else:
self.poller = IOWait()
for sock, address in self.streamer.get_sockets():
self.poller.watch(sock, read=True, write=False)
self._p = ioloop.PeriodicCallback(self._select, 1, io_loop=io_loop)
def start(self):
# starting the thread or greenlet
self._p.start()
super(SocketStatsCollector, self).start()
def stop(self):
self._p.stop()
BaseStatsCollector.stop(self)
def _select(self):
# polling for events
try:
events = self.poller.wait(_RESOLUTION)
except ValueError:
return
if len(events) == 0:
return
for socket, read, write in events:
if read:
self._rstats[socket.fileno()] += 1
def _aggregate(self, aggregate):
raise NotImplementedError()
def collect_stats(self):
# sending hits by sockets
sockets = self.streamer.get_sockets()
if len(sockets) == 0:
yield None
else:
fds = [(address, sock.fileno()) for sock, address in sockets]
total = {'addresses': [], 'reads': 0}
# we might lose a few hits here but it's ok
for address, fd in fds:
info = {}
info['fd'] = info['subtopic'] = fd
info['reads'] = self._rstats[fd]
total['reads'] += info['reads']
total['addresses'].append(address)
info['address'] = address
self._rstats[fd] = 0
yield info
yield total
class Connections(Base):
"""
Within SmartHome.py there is one instance of this class
The filenumber of a connection is the key to the contained dicts of
_connections and _servers
Additionally the filenumber is used for either epoll or kqueue depending
on the environment found for select.
A filenumber of value -1 is an error value.
"""
_connections = {}
_servers = {}
if hasattr(select, 'epoll'):
_ro = select.EPOLLIN | select.EPOLLHUP | select.EPOLLERR
_rw = _ro | select.EPOLLOUT
def __init__(self):
Base.__init__(self)
Base._poller = self
if hasattr(select, 'epoll'):
self._epoll = select.epoll()
elif hasattr(select, 'kqueue'):
self._kqueue = select.kqueue()
else:
logger.debug("Init connections using IOWait")
self._connections_found = 0
self._waitobj = IOWait()
def register_server(self, fileno, obj):
if fileno == -1:
logger.error("{} tried to register a server with filenumber == -1".format(obj))
return
self._servers[fileno] = obj
self._connections[fileno] = obj
if hasattr(select, 'epoll'):
self._epoll.register(fileno, self._ro)
elif hasattr(select, 'kqueue'):
event = [
select.kevent(fileno,
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD)
]
self._kqueue.control(event, 0, 0)
else:
logger.debug("register_server: put watch on fileno {}".format(fileno))
self._waitobj.watch(fileno, read=True)
def register_connection(self, fileno, obj):
if fileno == -1:
logger.error("tried to register a connection with filenumber == -1")
return
self._connections[fileno] = obj
if hasattr(select, 'epoll'):
self._epoll.register(fileno, self._ro)
elif hasattr(select, 'kqueue'):
event = [
select.kevent(fileno,
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD)
]
self._kqueue.control(event, 0, 0)
else:
logger.debug("register_connection: put watch on fileno {}".format(fileno))
self._waitobj.watch(fileno, read=True)
def unregister_connection(self, fileno):
if fileno == -1:
logger.error("tried to unregister a connection with filenumber == -1")
return
try:
if hasattr(select, 'epoll'):
self._epoll.unregister(fileno)
elif hasattr(select, 'kqueue'):
pass
else:
logger.debug("unregister_connection: unwatch fileno {}".format(fileno))
self._waitobj.unwatch(fileno)
except Exception as e:
logger.error("unregister a connection with filenumber == {} for epoll failed".format(fileno))
try:
del(self._connections[fileno])
except:
pass
try:
del(self._servers[fileno])
except:
pass
def monitor(self, obj):
self._monitor.append(obj)
def check(self):
for obj in self._monitor:
if not obj.connected:
obj.connect()
def trigger(self, fileno):
if fileno == -1:
logger.error("tried to trigger a connection with filenumber == -1")
return
if self._connections[fileno].outbuffer:
if hasattr(select, 'epoll'):
self._epoll.modify(fileno, self._rw)
elif hasattr(select, 'kqueue'):
event = [
select.kevent(fileno,
filter=select.KQ_FILTER_WRITE,
flags=select.KQ_EV_ADD | KQ_EV_ONESHOT)
]
self._kqueue.control(event, 0, 0)
else:
logger.error("trigger: Operating System without epoll or kqueue is currently not supported, please report this error")
def poll(self):
time.sleep(0.0000000001) # give epoll.modify a chance
if not self._connections:
time.sleep(1)
return
if -1 in self._connections:
logger.error("fileno -1 was found, please report to SmartHomeNG team")
del( self._connections[-1])
if hasattr(select, 'epoll') or hasattr(select, 'kqueue'):
for fileno in self._connections:
# Fix for: "RuntimeError: dictionary changed size during iteration"
#connections_keys = self._connections.keys()
#for fileno in connections_keys:
if fileno not in self._servers:
if hasattr(select, 'epoll'):
if self._connections[fileno].outbuffer:
try:
self._epoll.modify(fileno, self._rw)
except OSError as e:
# as with python 3.6 an OSError will be raised when a socket is already closed like with a settimeout
# the socket will need to be recreated then
logger.error("OSError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
except PermissionError as e:
logger.error("PermissionError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
except FileNotFoundError as e:
logger.error("FileNotFoundError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
else:
try:
self._epoll.modify(fileno, self._ro)
except OSError as e:
# as with python 3.6 an OSError will be raised when a socket is already closed like with a settimeout
# the socket will need to be recreated then
logger.error("OSError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
except PermissionError as e:
logger.error("PermissionError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
except FileNotFoundError as e:
logger.error("FileNotFoundError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
# here we could try to get rid of the connection that causes the headache
elif hasattr(select, 'kqueue'):
event = []
if self._connections[fileno].outbuffer:
event.append(select.kevent(fileno,
filter=select.KQ_FILTER_WRITE,
flags=select.KQ_EV_ADD | KQ_EV_ONESHOT))
else:
event.append(select.kevent(fileno,
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD))
self._kqueue.control(event, 0, 0)
if hasattr(select, 'epoll'):
for fileno, event in self._epoll.poll(timeout=1):
if fileno in self._servers:
server = self._servers[fileno]
server.handle_connection()
else:
if event & select.EPOLLIN:
try:
con = self._connections[fileno]
con._in()
except Exception as e:
con.close()
continue
if event & select.EPOLLOUT:
try:
con = self._connections[fileno]
con._out()
except Exception as e:
con.close()
continue
if event & (select.EPOLLHUP | select.EPOLLERR):
try:
con = self._connections[fileno]
con.close()
continue
except:
pass
elif hasattr(select, 'kqueue'):
for event in self._kqueue.control(None, 1):
fileno = event.ident
if fileno in self._servers:
server = self._servers[fileno]
server.handle_connection()
else:
if event.filter == select.KQ_FILTER_READ:
try:
con = self._connections[fileno]
con._in()
except Exception as e: # noqa
con.close()
continue
if event.filter == select.KQ_FILTER_WRITE:
try:
con = self._connections[fileno]
con._out()
except Exception as e: # noqa
con.close()
continue
if event.flags & select.KQ_EV_EOF:
try:
con = self._connections[fileno]
con.close()
continue
except:
pass
else:
# not using epoll or kqueue
n_connections = len(self._connections)
if self._connections_found != n_connections:
logger.debug("lib/connection.py poll() for len(self._connections)={}".format(n_connections))
self._connections_found = n_connections
watched = self._waitobj.get_watched()
nwatched = len(watched)
logger.debug("iowait in alpha status with {} watched connections".format(nwatched))
events = self._waitobj.wait()
nevents = len(events)
logger.debug("iowait reports {} events".format(nevents))
for fileno, read, write in events:
logger.debug("event for fileno={}, read={}, write={}".format(fileno, read, write))
if fileno in self._servers:
server = self._servers[fileno]
server.handle_connection()
else:
logger.debug("fileno {} not in self._servers".format(fileno))
if read:
try:
con = self._connections[fileno]
con._in()
except Exception as e: # noqa
con.close()
continue
if write:
try:
con = self._connections[fileno]
con._out()
except Exception as e: # noqa
con.close()
continue
def close(self):
if -1 in self._connections:
logger.error("Connections.close() tried to close a filenumber == -1")
try:
for fileno in self._connections:
try:
self._connections[fileno].close()
except:
pass
except:
pass
class SocketStatsCollector(BaseStatsCollector):
def __init__(self, streamer, name, callback_time=1., io_loop=None):
super(SocketStatsCollector, self).__init__(streamer, name,
callback_time, io_loop)
# if gevent is installed, we'll use a greenlet,
# otherwise we'll use a thread
try:
import gevent # NOQA
self.greenlet = True
except ImportError:
self.greenlet = False
self._rstats = defaultdict(int)
if self.greenlet:
self.poller = SelectIOWait()
else:
self.poller = IOWait()
for sock, address, fd in self.streamer.get_sockets():
self.poller.watch(sock, read=True, write=False)
self._p = ioloop.PeriodicCallback(self._select,
_LOOP_RES,
io_loop=io_loop)
def start(self):
# starting the thread or greenlet
self._p.start()
super(SocketStatsCollector, self).start()
def stop(self):
self._p.stop()
BaseStatsCollector.stop(self)
def _select(self):
# polling for events
try:
events = self.poller.wait(_RESOLUTION)
except ValueError:
return
if len(events) == 0:
return
for socket, read, write in events:
if read:
self._rstats[socket.fileno()] += 1
def _aggregate(self, aggregate):
raise NotImplementedError()
def collect_stats(self):
# sending hits by sockets
sockets = self.streamer.get_sockets()
if len(sockets) == 0:
yield None
else:
fds = [(address, sock.fileno(), fd)
for sock, address, fd in sockets]
total = {'addresses': [], 'reads': 0}
# we might lose a few hits here but it's ok
for address, monitored_fd, fd in fds:
info = {}
info['fd'] = info['subtopic'] = fd
info['reads'] = self._rstats[monitored_fd]
total['reads'] += info['reads']
total['addresses'].append(address)
info['address'] = address
self._rstats[monitored_fd] = 0
yield info
yield total
