Beispiel #1
0
    def __receive_thread_worker(self):
        waitobj = IOWait()
        waitobj.watch( self._socket, read=True)
        ### BMX poller = select.poll()
        ### BMX poller.register(self._socket, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR)
        __buffer = b''

        self._is_receiving = True
        self._receiving_callback and self._receiving_callback(self)
        while self._is_connected and self.__running:
            ### BMX events = poller.poll(1000)
            events = waitobj.wait(1000)     ### BMX
            ### BMX for fd, event in events:
            for fileno, read, write in events:  ### BMX
                ### BMX if event & select.POLLHUP:
                ### BMX     self.logger.warning("Client socket closed")
                # Check if POLLIN event triggered
                ### BMX if event & (select.POLLIN | select.POLLPRI):
                if read:
                    msg = self._socket.recv(4096)
                    # Check if incoming message is not empty
                    if msg:
                        # If we transfer in text mode decode message to string
                        if not self._binary:
                            msg = str.rstrip(str(msg, 'utf-8'))
                        # If we work in line mode (with a terminator) slice buffer into single chunks based on terminator
                        if self.terminator:
                            __buffer += msg
                            while True:
                                # terminator = int means fixed size chunks
                                if isinstance(self.terminator, int):
                                    i = self.terminator
                                    if i > len(__buffer):
                                        break
                                # terminator is str or bytes means search for it
                                else:
                                    i = __buffer.find(self.terminator)
                                    if i == -1:
                                        break
                                    i += len(self.terminator)
                                line = __buffer[:i]
                                __buffer = __buffer[i:]
                                if self._data_received_callback is not None:
                                    self._data_received_callback(self, line)
                        # If not in terminator mode just forward what we received
                        else:
                            if self._data_received_callback is not None:
                                self._data_received_callback(self, msg)
                    # If empty peer has closed the connection
                    else:
                        # Peer connection closed
                        self.logger.warning("Connection closed by peer {}".format(self._host))
                        self._is_connected = False
                        ### BMX poller.unregister()
                        waitobj.unwatch(self._socket)
                        self._disconnected_callback and self._disconnected_callback(self)
                        if self._autoreconnect:
                            self.logger.debug("Autoreconnect enabled for {}".format(self._host))
                            self.connect()
        self._is_receiving = False
Beispiel #2
0
class SocketStatsCollector(BaseStatsCollector):

    def __init__(self, streamer, name, callback_time=1., io_loop=None):
        super(SocketStatsCollector, self).__init__(streamer, name,
                callback_time, io_loop)
        # if gevent is installed, we'll use a greenlet,
        # otherwise we'll use a thread
        try:
            import gevent       # NOQA
            self.greenlet = True
        except ImportError:
            self.greenlet = False

        self._rstats = defaultdict(int)
        if self.greenlet:
            self.poller = SelectIOWait()
        else:
            self.poller = IOWait()

        for sock, address in self.streamer.get_sockets():
            self.poller.watch(sock, read=True, write=False)

        self._p = ioloop.PeriodicCallback(self._select, 1, io_loop=io_loop)

    def start(self):
        # starting the thread or greenlet
        self._p.start()
        super(SocketStatsCollector, self).start()

    def stop(self):
        self._p.stop()
        BaseStatsCollector.stop(self)

    def _select(self):
        # polling for events
        try:
            events = self.poller.wait(_RESOLUTION)
        except ValueError:
            return

        if len(events) == 0:
            return

        for socket, read, write in events:
            if read:
                self._rstats[socket.fileno()] += 1

    def _aggregate(self, aggregate):
        raise NotImplementedError()

    def collect_stats(self):
        # sending hits by sockets
        sockets = self.streamer.get_sockets()

        if len(sockets) == 0:
            yield None
        else:
            fds = [(address, sock.fileno()) for sock, address in sockets]
            total = {'addresses': [], 'reads': 0}

            # we might lose a few hits here but it's ok
            for address, fd in fds:
                info = {}
                info['fd'] = info['subtopic'] = fd
                info['reads'] = self._rstats[fd]
                total['reads'] += info['reads']
                total['addresses'].append(address)
                info['address'] = address
                self._rstats[fd] = 0
                yield info

            yield total
Beispiel #3
0
class Connections(Base):
    """
    Within SmartHome.py there is one instance of this class

    The filenumber of a connection is the key to the contained dicts of
    _connections and _servers
    Additionally the filenumber is used for either epoll or kqueue depending
    on the environment found for select.
    A filenumber of value -1 is an error value.
    """

    _connections = {}
    _servers = {}
    if hasattr(select, 'epoll'):
        _ro = select.EPOLLIN | select.EPOLLHUP | select.EPOLLERR
        _rw = _ro | select.EPOLLOUT

    def __init__(self):
        Base.__init__(self)
        Base._poller = self
        if hasattr(select, 'epoll'):
            self._epoll = select.epoll()
        elif hasattr(select, 'kqueue'):
            self._kqueue = select.kqueue()
        else:
            logger.debug("Init connections using IOWait")
            self._connections_found = 0
            self._waitobj = IOWait()

    def register_server(self, fileno, obj):
        if fileno == -1:
            logger.error("{} tried to register a server with filenumber == -1".format(obj))
            return
        self._servers[fileno] = obj
        self._connections[fileno] = obj
        if hasattr(select, 'epoll'):
            self._epoll.register(fileno, self._ro)
        elif hasattr(select, 'kqueue'):
            event = [
                select.kevent(fileno,
                       filter=select.KQ_FILTER_READ,
                       flags=select.KQ_EV_ADD)
            ]
            self._kqueue.control(event, 0, 0)
        else:
            logger.debug("register_server: put watch on fileno {}".format(fileno))
            self._waitobj.watch(fileno, read=True)

    def register_connection(self, fileno, obj):
        if fileno == -1:
            logger.error("tried to register a connection with filenumber == -1")
            return
        self._connections[fileno] = obj
        if hasattr(select, 'epoll'):
            self._epoll.register(fileno, self._ro)
        elif hasattr(select, 'kqueue'):
            event = [
                select.kevent(fileno,
                       filter=select.KQ_FILTER_READ,
                       flags=select.KQ_EV_ADD)
            ]
            self._kqueue.control(event, 0, 0)
        else:
            logger.debug("register_connection: put watch on fileno {}".format(fileno))
            self._waitobj.watch(fileno, read=True)

    def unregister_connection(self, fileno):
        if fileno == -1:
            logger.error("tried to unregister a connection with filenumber == -1")
            return
        try:
            if hasattr(select, 'epoll'):
                self._epoll.unregister(fileno)
            elif hasattr(select, 'kqueue'):
                pass
            else:
                logger.debug("unregister_connection: unwatch fileno {}".format(fileno))
                self._waitobj.unwatch(fileno)
        except Exception as e:
            logger.error("unregister a connection with filenumber == {} for epoll failed".format(fileno))

        try:
            del(self._connections[fileno])
        except:
            pass

        try:
            del(self._servers[fileno])
        except:
            pass

    def monitor(self, obj):
        self._monitor.append(obj)

    def check(self):
        for obj in self._monitor:
            if not obj.connected:
                obj.connect()

    def trigger(self, fileno):
        if fileno == -1:
            logger.error("tried to trigger a connection with filenumber == -1")
            return
        if self._connections[fileno].outbuffer:
            if hasattr(select, 'epoll'):
                self._epoll.modify(fileno, self._rw)
            elif hasattr(select, 'kqueue'):
                event = [
                    select.kevent(fileno,
                           filter=select.KQ_FILTER_WRITE,
                           flags=select.KQ_EV_ADD | KQ_EV_ONESHOT)
                ]
                self._kqueue.control(event, 0, 0)
            else:
                logger.error("trigger: Operating System without epoll or kqueue is currently not supported, please report this error")

    def poll(self):
        time.sleep(0.0000000001)  # give epoll.modify a chance
        if not self._connections:
            time.sleep(1)
            return

        if -1 in self._connections:
            logger.error("fileno -1 was found, please report to SmartHomeNG team")
            del( self._connections[-1])

        if hasattr(select, 'epoll') or hasattr(select, 'kqueue'):
            for fileno in self._connections:
            # Fix for: "RuntimeError: dictionary changed size during iteration"
            #connections_keys = self._connections.keys()
            #for fileno in connections_keys:
                if fileno not in self._servers:
                    if hasattr(select, 'epoll'):
                        if self._connections[fileno].outbuffer:
                            try:
                                self._epoll.modify(fileno, self._rw)
                            except OSError as e:
                                # as with python 3.6 an OSError will be raised when a socket is already closed like with a settimeout
                                # the socket will need to be recreated then
                                logger.error("OSError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache
                            except PermissionError as e:
                                logger.error("PermissionError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache
                            except FileNotFoundError as e:
                                logger.error("FileNotFoundError {} for epoll.modify(RW) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache
                        else:
                            try:
                                self._epoll.modify(fileno, self._ro)
                            except OSError as e:
                                # as with python 3.6 an OSError will be raised when a socket is already closed like with a settimeout
                                # the socket will need to be recreated then
                                logger.error("OSError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache
                            except PermissionError as e:
                                logger.error("PermissionError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache
                            except FileNotFoundError as e:
                                logger.error("FileNotFoundError {} for epoll.modify(RO) with fileno {} for object {}, please report to SmartHomeNG team".format(e, fileno, self._connections[fileno]))
                                # here we could try to get rid of the connection that causes the headache

                    elif hasattr(select, 'kqueue'):
                        event = []
                        if self._connections[fileno].outbuffer:
                            event.append(select.kevent(fileno,
                                                filter=select.KQ_FILTER_WRITE,
                                                flags=select.KQ_EV_ADD | KQ_EV_ONESHOT))
                        else:
                            event.append(select.kevent(fileno,
                                                filter=select.KQ_FILTER_READ,
                                                flags=select.KQ_EV_ADD))
                        self._kqueue.control(event, 0, 0)

            if hasattr(select, 'epoll'):
                for fileno, event in self._epoll.poll(timeout=1):
                    if fileno in self._servers:
                        server = self._servers[fileno]
                        server.handle_connection()
                    else:
                        if event & select.EPOLLIN:
                            try:
                                con = self._connections[fileno]
                                con._in()
                            except Exception as e:
                                con.close()
                                continue
                            if event & select.EPOLLOUT:
                                try:
                                    con = self._connections[fileno]
                                    con._out()
                                except Exception as e:
                                    con.close()
                                    continue
                            if event & (select.EPOLLHUP | select.EPOLLERR):
                                try:
                                    con = self._connections[fileno]
                                    con.close()
                                    continue
                                except:
                                    pass
            elif hasattr(select, 'kqueue'):
                for event in self._kqueue.control(None, 1):
                    fileno = event.ident
                    if fileno in self._servers:
                        server = self._servers[fileno]
                        server.handle_connection()
                    else:
                        if event.filter == select.KQ_FILTER_READ:
                            try:
                                con = self._connections[fileno]
                                con._in()
                            except Exception as e:  # noqa
                                con.close()
                                continue
                            if event.filter == select.KQ_FILTER_WRITE:
                                try:
                                    con = self._connections[fileno]
                                    con._out()
                                except Exception as e:  # noqa
                                    con.close()
                                    continue
                            if event.flags & select.KQ_EV_EOF:
                                try:
                                    con = self._connections[fileno]
                                    con.close()
                                    continue
                                except:
                                    pass
        else:
            # not using  epoll or kqueue
            n_connections = len(self._connections)
            if self._connections_found != n_connections:
                logger.debug("lib/connection.py poll() for len(self._connections)={}".format(n_connections))
                self._connections_found = n_connections

            watched = self._waitobj.get_watched()
            nwatched = len(watched)
            logger.debug("iowait in alpha status with {} watched connections".format(nwatched))
            events = self._waitobj.wait()
            nevents = len(events)
            logger.debug("iowait reports {} events".format(nevents))
            for fileno, read, write in events:
                logger.debug("event for fileno={}, read={}, write={}".format(fileno, read, write))
                if fileno in self._servers:
                    server = self._servers[fileno]
                    server.handle_connection()
                else:
                    logger.debug("fileno {} not in self._servers".format(fileno))
                    if read:
                        try:
                            con = self._connections[fileno]
                            con._in()
                        except Exception as e:  # noqa
                            con.close()
                            continue
                        if write:
                            try:
                                con = self._connections[fileno]
                                con._out()
                            except Exception as e:  # noqa
                                con.close()
                                continue


    def close(self):
        if -1 in self._connections:
            logger.error("Connections.close() tried to close a filenumber == -1")

        try:
            for fileno in self._connections:
                try:
                    self._connections[fileno].close()
                except:
                    pass
        except:
            pass
Beispiel #4
0
class SocketStatsCollector(BaseStatsCollector):
    def __init__(self, streamer, name, callback_time=1., io_loop=None):
        super(SocketStatsCollector, self).__init__(streamer, name,
                                                   callback_time, io_loop)
        # if gevent is installed, we'll use a greenlet,
        # otherwise we'll use a thread
        try:
            import gevent  # NOQA
            self.greenlet = True
        except ImportError:
            self.greenlet = False

        self._rstats = defaultdict(int)
        if self.greenlet:
            self.poller = SelectIOWait()
        else:
            self.poller = IOWait()

        for sock, address, fd in self.streamer.get_sockets():
            self.poller.watch(sock, read=True, write=False)

        self._p = ioloop.PeriodicCallback(self._select,
                                          _LOOP_RES,
                                          io_loop=io_loop)

    def start(self):
        # starting the thread or greenlet
        self._p.start()
        super(SocketStatsCollector, self).start()

    def stop(self):
        self._p.stop()
        BaseStatsCollector.stop(self)

    def _select(self):
        # polling for events
        try:
            events = self.poller.wait(_RESOLUTION)
        except ValueError:
            return

        if len(events) == 0:
            return

        for socket, read, write in events:
            if read:
                self._rstats[socket.fileno()] += 1

    def _aggregate(self, aggregate):
        raise NotImplementedError()

    def collect_stats(self):
        # sending hits by sockets
        sockets = self.streamer.get_sockets()

        if len(sockets) == 0:
            yield None
        else:
            fds = [(address, sock.fileno(), fd)
                   for sock, address, fd in sockets]
            total = {'addresses': [], 'reads': 0}

            # we might lose a few hits here but it's ok
            for address, monitored_fd, fd in fds:
                info = {}
                info['fd'] = info['subtopic'] = fd
                info['reads'] = self._rstats[monitored_fd]
                total['reads'] += info['reads']
                total['addresses'].append(address)
                info['address'] = address
                self._rstats[monitored_fd] = 0
                yield info

            yield total