コード例 #1
0
ファイル: context.py プロジェクト: tjguk/pyzo
class Context(object):
    """ Context(verbose=0, queue_params=None)
    
    A context represents a node in the network. It can connect to 
    multiple other contexts (using a yoton.Connection. 
    These other contexts can be in 
    another process on the same machine, or on another machine
    connected via a network or the internet.
    
    This class represents a context that can be used by channel instances
    to communicate to other channels in the network. (Thus the name.)
    
    The context is the entity that queue routes the packages produced 
    by the channels to the other context in the network, where
    the packages are distributed to the right channels. A context queues
    packages while it is not connected to any other context.
    
    If messages are send on a channel registered at this context while
    the context is not connected, the messages are stored by the
    context and will be send to the first connecting context.
    
    Example 1
    ---------
    # Create context and bind to a port on localhost
    context = yoton.Context()
    context.bind('localhost:11111')
    # Create a channel and send a message
    pub = yoton.PubChannel(context, 'test')
    pub.send('Hello world!')
    
    Example 2
    ---------
    # Create context and connect to the port on localhost
    context = yoton.Context()
    context.connect('localhost:11111')
    # Create a channel and receive a message
    sub = yoton.SubChannel(context, 'test')
    print(sub.recv() # Will print 'Hello world!'
    
    Queue params
    ------------
    The queue_params parameter allows one to specify the package queues
    used in the system. It is recommended to use the same parameters
    for every context in the network. The value of queue_params should
    be a 2-element tuple specifying queue size and discard mode. The
    latter can be 'old' (default) or 'new', meaning that if the queue
    is full, either the oldest or newest messages are discarted.
    
    """
    def __init__(self, verbose=0, queue_params=None):

        # Whether or not to write status information
        self._verbose = verbose

        # Store queue parameters
        if queue_params is None:
            queue_params = BUF_MAX_LEN, 'old'
        if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
            raise ValueError('queue_params should be a 2-element tuple.')
        self._queue_params = queue_params

        # Create unique key to identify this context
        self._id = UID().get_int()

        # Channels currently registered. Maps slots to channel instance.
        self._sending_channels = {}
        self._receiving_channels = {}

        # The list of connections
        self._connections = []
        self._connections_lock = threading.RLock()

        # Queue used during startup to collect packages
        # This queue is also subject to the _connections_lock
        self._startupQueue = PackageQueue(*queue_params)

        # For numbering and routing the packages
        self._send_seq = 0
        self._recv_seq = 0
        self._source_map = {}

    def close(self):
        """ close()
        
        Close the context in a nice way, by closing all connections
        and all channels.
        
        Closing a connection means disconnecting two contexts. Closing
        a channel means disasociating a channel from its context. 
        Unlike connections and channels, a Context instance can be reused 
        after closing (although this might not always the best strategy).
        
        """

        # Close all connections (also the waiting connections!)
        for c in self.connections_all:
            c.close('Closed by the context.')

        # Close all channels
        self.close_channels()

    def close_channels(self):
        """ close_channels()
        
        Close all channels associated with this context. This does
        not close the connections. See also close().
        
        """

        # Get all channels
        channels1 = [c for c in self._sending_channels.values()]
        channels2 = [c for c in self._receiving_channels.values()]

        # Close all channels
        for c in set(channels1 + channels2):
            c.close()

    ## Properties

    @property
    def connections_all(self):
        """ Get a list of all Connection instances currently
        associated with this context, including pending connections 
        (connections waiting for another end to connect).
        In addition to normal list indexing, the connections objects can be
        queried from this list using their name.
        """

        # Lock
        self._connections_lock.acquire()
        try:
            return [c for c in self._connections if c.is_alive]
        finally:
            self._connections_lock.release()

    @property
    def connections(self):
        """ Get a list of the Connection instances currently
        active for this context. 
        In addition to normal list indexing, the connections objects can be
        queried  from this list using their name.
        """
        # Lock
        self._connections_lock.acquire()

        try:

            # Clean up any dead connections
            copy = ConnectionCollection()
            to_remove = []
            for c in self._connections:
                if not c.is_alive:
                    to_remove.append(c)
                elif c.is_connected:
                    copy.append(c)

            # Clean
            for c in to_remove:
                self._connections.remove(c)

            # Return copy
            return copy

        finally:
            self._connections_lock.release()

    @property
    def connection_count(self):
        """ Get the number of connected contexts. Can be used as a boolean
        to check if the context is connected to any other context.
        """
        return len(self.connections)

    @property
    def id(self):
        """ The 8-byte UID of this context.
        """
        return self._id

    ## Public methods

    def bind(self, address, max_tries=1, name=''):
        """ bind(address, max_tries=1, name='')
        
        Setup a connection with another Context, by being the host.
        This method starts a thread that waits for incoming connections.
        Error messages are printed when an attemped connect fails. the
        thread keeps trying until a successful connection is made, or until
        the connection is closed.
        
        Returns a Connection instance that represents the
        connection to the other context. These connection objects 
        can also be obtained via the Context.connections property.
        
        Parameters
        ----------
        address : str
            Should be of the shape hostname:port. The port should be an
            integer number between 1024 and 2**16. If port does not 
            represent a number, a valid port number is created using a 
            hash function.
        max_tries : int
            The number of ports to try; starting from the given port, 
            subsequent ports are tried until a free port is available. 
            The final port can be obtained using the 'port' property of
            the returned Connection instance.
        name : string
            The name for the created Connection instance. It can
            be used as a key in the connections property.
        
        Notes on hostname
        -----------------
        The hostname can be:
          * The IP address, or the string hostname of this computer. 
          * 'localhost': the connections is only visible from this computer. 
            Also some low level networking layers are bypassed, which results
            in a faster connection. The other context should also connect to
            'localhost'.
          * 'publichost': the connection is visible by other computers on the 
            same network. Optionally an integer index can be appended if
            the machine has multiple IP addresses (see socket.gethostbyname_ex).
        
        """

        # Trigger cleanup of closed connections
        self.connections

        # Split address in protocol, real hostname and port number
        protocol, hostname, port = split_address(address)

        # Based on protocol, instantiate connection class (currently only tcp)
        if False:  #protocol == 'itc':
            connection = ItcConnection(self, name)
        else:
            connection = TcpConnection(self, name)

        # Bind connection
        connection._bind(hostname, port, max_tries)

        # Save connection instance
        self._connections_lock.acquire()
        try:
            # Push packages from startup queue
            while len(self._startupQueue):
                connection._inject_package(self._startupQueue.pop())
            # Add connection object to list of connections
            self._connections.append(connection)
        finally:
            self._connections_lock.release()

        # Return Connection instance
        return connection

    def connect(self, address, timeout=1.0, name=''):
        """ connect(self, address, timeout=1.0, name='')
        
        Setup a connection with another context, by connection to a 
        hosting context. An error is raised when the connection could
        not be made.
        
        Returns a Connection instance that represents the
        connection to the other context. These connection objects 
        can also be obtained via the Context.connections property.
        
        Parameters
        ----------
        address : str
            Should be of the shape hostname:port. The port should be an
            integer number between 1024 and 2**16. If port does not 
            represent a number, a valid port number is created using a 
            hash function.
        max_tries : int
            The number of ports to try; starting from the given port, 
            subsequent ports are tried until a free port is available. 
            The final port can be obtained using the 'port' property of
            the returned Connection instance.
        name : string
            The name for the created Connection instance. It can
            be used as a key in the connections property.
        
        Notes on hostname
        -----------------
        The hostname can be:
          * The IP address, or the string hostname of this computer. 
          * 'localhost': the connection is only visible from this computer. 
            Also some low level networking layers are bypassed, which results
            in a faster connection. The other context should also host as
            'localhost'.
          * 'publichost': the connection is visible by other computers on the 
            same network. Optionally an integer index can be appended if
            the machine has multiple IP addresses (see socket.gethostbyname_ex).
        
        """

        # Trigger cleanup of closed connections
        self.connections

        # Split address in protocol, real hostname and port number
        protocol, hostname, port = split_address(address)

        # Based on protocol, instantiate connection class (currently only tcp)
        if False:  #protocol == 'itc':
            connection = ItcConnection(self, name)
        else:
            connection = TcpConnection(self, name)

        # Create new connection and connect it
        connection._connect(hostname, port, timeout)

        # Save connection instance
        self._connections_lock.acquire()
        try:
            # Push packages from startup queue
            while self._startupQueue:
                connection._inject_package(self._startupQueue.pop())
            # Add connection object to list of connections
            self._connections.append(connection)
        finally:
            self._connections_lock.release()

        # Send message in the network to signal a new connection
        bb = 'NEW_CONNECTION'.encode('utf-8')
        p = Package(bb, SLOT_CONTEXT, self._id, 0, 0, 0, 0)
        self._send_package(p)

        # Return Connection instance
        return connection

    def flush(self, timeout=5.0):
        """ flush(timeout=5.0)
        
        Wait until all pending messages are send. This will flush all
        messages posted from the calling thread. However, it is not
        guaranteed that no new messages are posted from another thread.
        
        Raises an error when the flushing times out.
        
        """
        # Flush all connections
        for c in self.connections:
            c.flush(timeout)

        # Done (backward compatibility)
        return True

    ## Private methods used by the Channel classes

    def _register_sending_channel(self, channel, slot, slotname=''):
        """ _register_sending_channel(channel, slot, slotname='')
        
        The channel objects use this method to register themselves 
        at a particular slot.
        
        """

        # Check if this slot is free
        if slot in self._sending_channels:
            raise ValueError("Slot not free: " + str(slotname))

        # Register
        self._sending_channels[slot] = channel

    def _register_receiving_channel(self, channel, slot, slotname=''):
        """ _register_receiving_channel(channel, slot, slotname='')
        
        The channel objects use this method to register themselves 
        at a particular slot.
        
        """

        # Check if this slot is free
        if slot in self._receiving_channels:
            raise ValueError("Slot not free: " + str(slotname))

        # Register
        self._receiving_channels[slot] = channel

    def _unregister_channel(self, channel):
        """ _unregister_channel(channel)
        
        Unregisters the given channel. That channel can no longer
        receive messages, and should no longer send messages.
        
        """
        for D in [self._receiving_channels, self._sending_channels]:
            for key in [key for key in D.keys()]:
                if D[key] == channel:
                    D.pop(key)

    ## Private methods to pass packages between context and io-threads

    def _send_package(self, package):
        """ _send_package(package)
        
        Used by the channels to send a package into the network.
        This method routes the package to all currentlt connected
        connections. If there are none, the packages is queued at
        the context.
        
        """

        # Add number
        self._send_seq += 1
        package._source_seq = self._send_seq

        # Send to all connections, or queue if there are none
        self._connections_lock.acquire()
        try:
            ok = False
            for c in self._connections:
                if c.is_alive:  # Waiting or connected
                    c._send_package(package)
                    ok = True
            # Should we queue the package?
            if not ok:
                self._startupQueue.push(package)
        finally:
            self._connections_lock.release()

    def _recv_package(self, package, connection):
        """ _recv_package(package, connection)
        
        Used by the connections to receive a package at this
        context. The package is distributed to all connections
        except the calling one. The package is also distributed
        to the right channel (if applicable).
        
        """

        # Get slot
        slot = package._slot

        # Init what to do with the package
        send_further = False
        deposit_here = False

        # Get what to do with the package
        last_seq = self._source_map.get(package._source_id, 0)
        if last_seq < package._source_seq:
            # Update source map
            self._source_map[package._source_id] = package._source_seq
            if package._dest_id == 0:
                # Add to both lists, first attach seq nr
                self._recv_seq += 1
                package._recv_seq = self._recv_seq
                send_further, deposit_here = True, True
            elif package._dest_id == self._id:
                # Add only to process list, first attach seq nr
                self._recv_seq += 1
                package._recv_seq = self._recv_seq
                deposit_here = True
            else:
                # Send package to connected nodes
                send_further = True

        # Send package to other context (over all alive connections)
        if send_further:
            self._connections_lock.acquire()
            try:
                for c in self._connections:
                    if c is connection or not c.is_alive:
                        continue
                    c._send_package(package)
            finally:
                self._connections_lock.release()

        # Process package here or pass to channel
        if deposit_here:
            if slot == SLOT_CONTEXT:
                # Context-to-context messaging;
                # A slot starting with a space reprsents the context
                self._recv_context_package(package)
            else:
                # Give package to a channel (if applicable)
                channel = self._receiving_channels.get(slot, None)
                if channel is not None:
                    channel._recv_package(package)

    def _recv_context_package(self, package):
        """ _recv_context_package(package)
        
        Process a package addressed at the context itself. This is how
        the context handles higher-level connection tasks.
        
        """

        # Get message: context messages are always utf-8 encoded strings
        message = package._data.decode('utf-8')

        if message == 'CLOSE_CONNECTION':
            # Close the connection. Check which one of our connections is
            # connected with the context that send this message.
            self._connections_lock.acquire()
            try:
                for c in self.connections:
                    if c.is_connected and c.id2 == package._source_id:
                        c.close(connection.STOP_CLOSED_FROM_THERE, False)
            finally:
                self._connections_lock.release()

        elif message == 'NEW_CONNECTION':
            # Resend all status channels
            for channel in self._sending_channels.values():
                if hasattr(channel, '_current_message') and hasattr(
                        channel, 'send_last'):
                    channel.send_last()

        else:
            print('Yoton: Received unknown context message: ' + message)
コード例 #2
0
ファイル: context.py プロジェクト: BrenBarn/pyzo
class Context(object):
    """ Context(verbose=0, queue_params=None)
    
    A context represents a node in the network. It can connect to 
    multiple other contexts (using a yoton.Connection. 
    These other contexts can be in 
    another process on the same machine, or on another machine
    connected via a network or the internet.
    
    This class represents a context that can be used by channel instances
    to communicate to other channels in the network. (Thus the name.)
    
    The context is the entity that queue routes the packages produced 
    by the channels to the other context in the network, where
    the packages are distributed to the right channels. A context queues
    packages while it is not connected to any other context.
    
    If messages are send on a channel registered at this context while
    the context is not connected, the messages are stored by the
    context and will be send to the first connecting context.
    
    Example 1
    ---------
    # Create context and bind to a port on localhost
    context = yoton.Context()
    context.bind('localhost:11111')
    # Create a channel and send a message
    pub = yoton.PubChannel(context, 'test')
    pub.send('Hello world!')
    
    Example 2
    ---------
    # Create context and connect to the port on localhost
    context = yoton.Context()
    context.connect('localhost:11111')
    # Create a channel and receive a message
    sub = yoton.SubChannel(context, 'test')
    print(sub.recv() # Will print 'Hello world!'
    
    Queue params
    ------------
    The queue_params parameter allows one to specify the package queues
    used in the system. It is recommended to use the same parameters
    for every context in the network. The value of queue_params should
    be a 2-element tuple specifying queue size and discard mode. The
    latter can be 'old' (default) or 'new', meaning that if the queue
    is full, either the oldest or newest messages are discarted.
    
    """
    
    def __init__(self, verbose=0, queue_params=None):
        
        # Whether or not to write status information
        self._verbose = verbose
        
        # Store queue parameters
        if queue_params is None:
            queue_params = BUF_MAX_LEN, 'old'
        if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
            raise ValueError('queue_params should be a 2-element tuple.')
        self._queue_params = queue_params
        
        # Create unique key to identify this context
        self._id = UID().get_int()
        
        # Channels currently registered. Maps slots to channel instance.
        self._sending_channels = {}
        self._receiving_channels = {}
        
        # The list of connections
        self._connections = []
        self._connections_lock = threading.RLock()
        
        # Queue used during startup to collect packages
        # This queue is also subject to the _connections_lock
        self._startupQueue = PackageQueue(*queue_params)
        
        # For numbering and routing the packages
        self._send_seq = 0
        self._recv_seq = 0
        self._source_map = {}
    
    
    def close(self):
        """ close()
        
        Close the context in a nice way, by closing all connections
        and all channels.
        
        Closing a connection means disconnecting two contexts. Closing
        a channel means disasociating a channel from its context. 
        Unlike connections and channels, a Context instance can be reused 
        after closing (although this might not always the best strategy).
        
        """
        
        # Close all connections (also the waiting connections!)
        for c in self.connections_all:
            c.close('Closed by the context.')
        
        # Close all channels
        self.close_channels()
    
    
    def close_channels(self):
        """ close_channels()
        
        Close all channels associated with this context. This does
        not close the connections. See also close().
        
        """
        
        # Get all channels
        channels1 = [c for c in self._sending_channels.values()]
        channels2 = [c for c in self._receiving_channels.values()]
        
        # Close all channels
        for c in set(channels1+channels2):
            c.close()
    
    
    ## Properties
    
    @property
    def connections_all(self):
        """ Get a list of all Connection instances currently
        associated with this context, including pending connections 
        (connections waiting for another end to connect).
        In addition to normal list indexing, the connections objects can be
        queried from this list using their name.
        """
        
        # Lock
        self._connections_lock.acquire()
        try:
            return [c for c in self._connections if c.is_alive]
        finally:
            self._connections_lock.release()
    
    
    @property
    def connections(self):
        """ Get a list of the Connection instances currently
        active for this context. 
        In addition to normal list indexing, the connections objects can be
        queried  from this list using their name.
        """        
        # Lock
        self._connections_lock.acquire()
        
        try:
            
            # Clean up any dead connections
            copy = ConnectionCollection()
            to_remove = []
            for c in self._connections:
                if not c.is_alive:
                    to_remove.append(c)
                elif c.is_connected:
                    copy.append(c)
            
            # Clean
            for c in to_remove:
                self._connections.remove(c)
            
            # Return copy
            return copy
        
        finally:
            self._connections_lock.release()
    
    
    @property
    def connection_count(self):
        """ Get the number of connected contexts. Can be used as a boolean
        to check if the context is connected to any other context.
        """
        return len(self.connections)
    
    
    @property
    def id(self):
        """ The 8-byte UID of this context.
        """
        return self._id
    
    
    ## Public methods
    
    
    def bind(self, address, max_tries=1, name=''):
        """ bind(address, max_tries=1, name='')
        
        Setup a connection with another Context, by being the host.
        This method starts a thread that waits for incoming connections.
        Error messages are printed when an attemped connect fails. the
        thread keeps trying until a successful connection is made, or until
        the connection is closed.
        
        Returns a Connection instance that represents the
        connection to the other context. These connection objects 
        can also be obtained via the Context.connections property.
        
        Parameters
        ----------
        address : str
            Should be of the shape hostname:port. The port should be an
            integer number between 1024 and 2**16. If port does not 
            represent a number, a valid port number is created using a 
            hash function.
        max_tries : int
            The number of ports to try; starting from the given port, 
            subsequent ports are tried until a free port is available. 
            The final port can be obtained using the 'port' property of
            the returned Connection instance.
        name : string
            The name for the created Connection instance. It can
            be used as a key in the connections property.
        
        Notes on hostname
        -----------------
        The hostname can be:
          * The IP address, or the string hostname of this computer. 
          * 'localhost': the connections is only visible from this computer. 
            Also some low level networking layers are bypassed, which results
            in a faster connection. The other context should also connect to
            'localhost'.
          * 'publichost': the connection is visible by other computers on the 
            same network. Optionally an integer index can be appended if
            the machine has multiple IP addresses (see socket.gethostbyname_ex).
        
        """ 
        
        # Trigger cleanup of closed connections
        self.connections
        
        # Split address in protocol, real hostname and port number
        protocol, hostname, port = split_address(address)
        
        # Based on protocol, instantiate connection class (currently only tcp)
        if False:#protocol == 'itc':
            connection = ItcConnection(self, name)
        else:
            connection = TcpConnection(self, name)
        
        # Bind connection
        connection._bind(hostname, port, max_tries)
        
        # Save connection instance
        self._connections_lock.acquire()
        try:
            # Push packages from startup queue
            while len(self._startupQueue):
                connection._inject_package(self._startupQueue.pop())
            # Add connection object to list of connections
            self._connections.append(connection)
        finally:
            self._connections_lock.release()
        
        # Return Connection instance
        return connection
    
    
    def connect(self, address, timeout=1.0, name=''):
        """ connect(self, address, timeout=1.0, name='')
        
        Setup a connection with another context, by connection to a 
        hosting context. An error is raised when the connection could
        not be made.
        
        Returns a Connection instance that represents the
        connection to the other context. These connection objects 
        can also be obtained via the Context.connections property.
        
        Parameters
        ----------
        address : str
            Should be of the shape hostname:port. The port should be an
            integer number between 1024 and 2**16. If port does not 
            represent a number, a valid port number is created using a 
            hash function.
        max_tries : int
            The number of ports to try; starting from the given port, 
            subsequent ports are tried until a free port is available. 
            The final port can be obtained using the 'port' property of
            the returned Connection instance.
        name : string
            The name for the created Connection instance. It can
            be used as a key in the connections property.
        
        Notes on hostname
        -----------------
        The hostname can be:
          * The IP address, or the string hostname of this computer. 
          * 'localhost': the connection is only visible from this computer. 
            Also some low level networking layers are bypassed, which results
            in a faster connection. The other context should also host as
            'localhost'.
          * 'publichost': the connection is visible by other computers on the 
            same network. Optionally an integer index can be appended if
            the machine has multiple IP addresses (see socket.gethostbyname_ex).
        
        """
        
        # Trigger cleanup of closed connections
        self.connections
        
        # Split address in protocol, real hostname and port number
        protocol, hostname, port = split_address(address)
        
        # Based on protocol, instantiate connection class (currently only tcp)
        if False:#protocol == 'itc':
            connection = ItcConnection(self, name)
        else:
            connection = TcpConnection(self, name)
        
        # Create new connection and connect it
        connection._connect(hostname, port, timeout)
        
        # Save connection instance
        self._connections_lock.acquire()
        try:
            # Push packages from startup queue
            while self._startupQueue:
                connection._inject_package(self._startupQueue.pop())
            # Add connection object to list of connections
            self._connections.append(connection)
        finally:
            self._connections_lock.release()
        
        # Send message in the network to signal a new connection
        bb = 'NEW_CONNECTION'.encode('utf-8')
        p = Package(bb, SLOT_CONTEXT, self._id, 0,0,0,0)
        self._send_package(p)
        
        # Return Connection instance
        return connection
    
    
    def flush(self, timeout=5.0):
        """ flush(timeout=5.0)
        
        Wait until all pending messages are send. This will flush all
        messages posted from the calling thread. However, it is not
        guaranteed that no new messages are posted from another thread.
        
        Raises an error when the flushing times out.
        
        """
        # Flush all connections
        for c in self.connections:
            c.flush(timeout)
        
        # Done (backward compatibility)
        return True
    
    
    ## Private methods used by the Channel classes
    
    
    def _register_sending_channel(self, channel, slot, slotname=''):
        """ _register_sending_channel(channel, slot, slotname='')
        
        The channel objects use this method to register themselves 
        at a particular slot.
        
        """ 
        
        # Check if this slot is free
        if slot in self._sending_channels:
            raise ValueError("Slot not free: " + str(slotname))
        
        # Register
        self._sending_channels[slot] = channel
    
    
    def _register_receiving_channel(self, channel, slot, slotname=''):
        """ _register_receiving_channel(channel, slot, slotname='')
        
        The channel objects use this method to register themselves 
        at a particular slot.
        
        """ 
        
        # Check if this slot is free
        if slot in self._receiving_channels:
            raise ValueError("Slot not free: " + str(slotname))
        
        # Register
        self._receiving_channels[slot] = channel
    
    
    def _unregister_channel(self, channel):
        """ _unregister_channel(channel)
        
        Unregisters the given channel. That channel can no longer
        receive messages, and should no longer send messages.
        
        """
        for D in [self._receiving_channels, self._sending_channels]:            
            for key in [key for key in D.keys()]:
                if D[key] == channel:
                    D.pop(key)
    
    
    ## Private methods to pass packages between context and io-threads
    
    
    def _send_package(self, package):
        """ _send_package(package)
        
        Used by the channels to send a package into the network.
        This method routes the package to all currentlt connected
        connections. If there are none, the packages is queued at
        the context.
        
        """
        
        # Add number
        self._send_seq += 1
        package._source_seq = self._send_seq
        
        # Send to all connections, or queue if there are none
        self._connections_lock.acquire()
        try:
            ok = False
            for c in self._connections:
                if c.is_alive: # Waiting or connected
                    c._send_package(package)
                    ok = True
            # Should we queue the package?
            if not ok:
                self._startupQueue.push(package)
        finally:
            self._connections_lock.release()
    
    
    def _recv_package(self, package, connection):
        """ _recv_package(package, connection)
        
        Used by the connections to receive a package at this
        context. The package is distributed to all connections
        except the calling one. The package is also distributed
        to the right channel (if applicable).
        
        """
        
        # Get slot
        slot = package._slot
        
        # Init what to do with the package
        send_further = False
        deposit_here = False
        
        # Get what to do with the package
        last_seq = self._source_map.get(package._source_id, 0)
        if last_seq < package._source_seq:
            # Update source map
            self._source_map[package._source_id] = package._source_seq
            if package._dest_id == 0:
                # Add to both lists, first attach seq nr
                self._recv_seq += 1
                package._recv_seq = self._recv_seq
                send_further, deposit_here = True, True
            elif package._dest_id == self._id:
                # Add only to process list, first attach seq nr
                self._recv_seq += 1
                package._recv_seq = self._recv_seq
                deposit_here = True
            else:
                # Send package to connected nodes
                send_further = True
        
        
        # Send package to other context (over all alive connections)
        if send_further:
            self._connections_lock.acquire()
            try:
                for c in self._connections:
                    if c is connection or not c.is_alive:
                        continue
                    c._send_package(package)
            finally:
                self._connections_lock.release()
        
        
        # Process package here or pass to channel
        if deposit_here:
            if slot == SLOT_CONTEXT:
                # Context-to-context messaging;
                # A slot starting with a space reprsents the context 
                self._recv_context_package(package)
            else:
                # Give package to a channel (if applicable)
                channel = self._receiving_channels.get(slot, None)
                if channel is not None:
                    channel._recv_package(package)
    
    
    def _recv_context_package(self, package):
        """ _recv_context_package(package)
        
        Process a package addressed at the context itself. This is how
        the context handles higher-level connection tasks.
        
        """ 
        
        # Get message: context messages are always utf-8 encoded strings
        message = package._data.decode('utf-8')
        
        if message == 'CLOSE_CONNECTION':
            # Close the connection. Check which one of our connections is
            # connected with the context that send this message.
            self._connections_lock.acquire()
            try:
                for c in self.connections:
                    if c.is_connected and c.id2 == package._source_id:
                        c.close(connection.STOP_CLOSED_FROM_THERE, False)
            finally:
                self._connections_lock.release()
        
        elif message == 'NEW_CONNECTION':
            # Resend all status channels
            for channel in self._sending_channels.values():
                if hasattr(channel, '_current_message') and hasattr(channel, 'send_last'):
                    channel.send_last()
        
        else:
            print('Yoton: Received unknown context message: '+message)
コード例 #3
0
class BaseChannel(object):
    """BaseChannel(context, slot_base, message_type=yoton.TEXT)

    Abstract class for all channels.

    Parameters
    ----------
    context : yoton.Context instance
        The context that this channel uses to send messages in a network.
    slot_base : string
        The base slot name. The channel appends an extension to indicate
        message type and messaging pattern to create the final slot name.
        The final slot is used to connect channels at different contexts
        in a network
    message_type : yoton.MessageType instance
        (default is yoton.TEXT)
        Object to convert messages to bytes and bytes to messages.
        Users can create their own message_type class to enable
        communicating any type of message they want.

    Details
    -------
    Messages send via a channel are delivered asynchronically to the
    corresponding channels.

    All channels are associated with a context and can be used to send
    messages to other channels in the network. Each channel is also
    associated with a slot, which is a string that represents a kind
    of address. A message send by a channel at slot X can only be received
    by a channel with slot X.

    Note that the channel appends an extension
    to the user-supplied slot name, that represents the message type
    and messaging pattern of the channel. In this way, it is prevented
    that for example a PubChannel can communicate with a RepChannel.

    """
    def __init__(self, context, slot_base, message_type=None):

        # Store context
        if not isinstance(context, Context):
            raise ValueError("Context not valid.")
        self._context = context

        # Check message type
        if message_type is None:
            message_type = TEXT
        if isinstance(message_type, type) and issubclass(
                message_type, MessageType):
            message_type = message_type()
        if isinstance(message_type, MessageType):
            message_type = message_type
        else:
            raise ValueError("message_type should be a MessageType instance.")

        # Store message type and conversion methods
        self._message_type_instance = message_type
        self.message_from_bytes = message_type.message_from_bytes
        self.message_to_bytes = message_type.message_to_bytes

        # Queue for incoming trafic (not used for pure sending channels)
        self._q_in = PackageQueue(*context._queue_params)

        # For sending channels: to lock the channel for sending
        self._send_condition = threading.Condition()
        self._is_send_locked = 0  # "True" is the timeout time

        # Signal for receiving data
        self._received_signal = yoton.events.Signal()
        self._posted_received_event = False

        # Channels can be closed
        self._closed = False

        # Event driven mode
        self._run_mode = 0

        # Init slots
        self._init_slots(slot_base)

    def _init_slots(self, slot_base):
        """_init_slots(slot_base)

        Called from __init__ to initialize the slots and perform all checks.

        """

        # Check if slot is string
        if not isinstance(slot_base, basestring):
            raise ValueError("slot_base must be a string.")

        # Get full slot names, init byte versions
        slots_t = []
        slots_h = []

        # Get extension for message type and messaging pattern
        ext_type = self._message_type_instance.message_type_name()
        ext_patterns = self._messaging_patterns()  # (incoming, outgoing)

        # Normalize and check slot names
        for ext_pattern in ext_patterns:
            if not ext_pattern:
                slots_t.append(None)
                slots_h.append(0)
                continue
            # Get full name
            slot = slot_base + "." + ext_type + "." + ext_pattern
            # Store text version
            slots_t.append(slot)
            # Strip and make lowercase
            slot = slot.strip().lower()
            # Hash
            slots_h.append(slot_hash(slot))

        # Store slots
        self._slot_out = slots_t[0]
        self._slot_in = slots_t[1]
        self._slot_out_h = slots_h[0]
        self._slot_in_h = slots_h[1]

        # Register slots (warn if neither slot is valid)
        if self._slot_out_h:
            self._context._register_sending_channel(self, self._slot_out_h,
                                                    self._slot_out)
        if self._slot_in_h:
            self._context._register_receiving_channel(self, self._slot_in_h,
                                                      self._slot_in)
        if not self._slot_out_h and not self._slot_in_h:
            raise ValueError("This channel does not have valid slots.")

    def _messaging_patterns(self):
        """_messaging_patterns()

        Implement to return a string that specifies the pattern
        for sending and receiving, respecitively.

        """
        raise NotImplementedError()

    def close(self):
        """close()

        Close the channel, i.e. unregisters this channel at the context.
        A closed channel cannot be reused.

        Future attempt to send() messages will result in an IOError
        being raised. Messages currently in the channel's queue can
        still be recv()'ed, but no new messages will be delivered at
        this channel.

        """
        # We keep a reference to the context, otherwise we need locks
        # The context clears the reference to this channel when unregistering.
        self._closed = True
        self._context._unregister_channel(self)

    def _send(self, message, dest_id=0, dest_seq=0):
        """_send(message, dest_id=0, dest_seq=0)

        Sends a message of raw bytes without checking whether they're bytes.
        Optionally, dest_id and dest_seq represent the message that
        this message  replies to. These are used for the request/reply
        pattern.

        Returns the package that will be send (or None). The context
        will set _source_id on the package right before
        sending it away.

        """

        # Check if still open
        if self._closed:
            className = self.__class__.__name__
            raise IOError("Cannot send from closed %s %i." %
                          (className, id(self)))

        if message:
            # If send_locked, wait at most one second
            if self._is_send_locked:
                self._send_condition.acquire()
                try:
                    self._send_condition.wait(1.0)  # wait for notify
                finally:
                    self._send_condition.release()
                    if time.time() > self._is_send_locked:
                        self._is_send_locked = 0
            # Push it on the queue as a package
            slot = self._slot_out_h
            cid = self._context._id
            p = Package(message, slot, cid, 0, dest_id, dest_seq, 0)
            self._context._send_package(p)
            # Return package
            return p
        else:
            return None

    def _recv(self, block):
        """_recv(block)

        Receive a package (or None).

        """

        if block is True:
            # Block for 0.25 seconds so that KeyboardInterrupt works
            while not self._closed:
                try:
                    return self._q_in.pop(0.25)
                except self._q_in.Empty:
                    continue

        else:
            # Block normal
            try:
                return self._q_in.pop(block)
            except self._q_in.Empty:
                return None

    def _set_send_lock(self, value):
        """_set_send_lock(self, value)

        Set or unset the blocking for the _send() method.

        """
        # Set send lock variable. We adopt a timeout (10s) just in case
        # the SubChannel that locks the PubChannel gets disconnected and
        # is unable to unlock it.
        if value:
            self._is_send_locked = time.time() + 10.0
        else:
            self._is_send_locked = 0
        # Notify any threads that are waiting in _send()
        if not value:
            self._send_condition.acquire()
            try:
                self._send_condition.notifyAll()
            finally:
                self._send_condition.release()

    ## How packages are inserted in this channel for receiving

    def _inject_package(self, package):
        """_inject_package(package)

        Same as _recv_package, but by definition do not block.
        _recv_package is overloaded in SubChannel. _inject_package is not.

        """
        self._q_in.push(package)
        self._maybe_emit_received()

    def _recv_package(self, package):
        """_recv_package(package)

        Put package in the queue.

        """
        self._q_in.push(package)
        self._maybe_emit_received()

    def _maybe_emit_received(self):
        """_maybe_emit_received()

        We want to emit a signal, but in such a way that multiple
        arriving packages result in a single emit. This methods
        only posts an event if it has not been done, or if the previous
        event has been handled.

        """
        if not self._posted_received_event:
            self._posted_received_event = True
            event = yoton.events.Event(self._emit_received)
            yoton.app.post_event(event)

    def _emit_received(self):
        """_emit_received()

        Emits the "received" signal. This method is called once new data
        has been received. However, multiple arrived messages may
        result in a single call to this method. There is also no
        guarantee that recv() has not been called in the mean time.

        Also sets the variabele so that a new event for this may be
        created. This method is called from the event loop.

        """
        self._posted_received_event = False  # Reset
        self.received.emit_now(self)

    # Received property sits on the BaseChannel because is is used by almost
    # all channels. Note that PubChannels never emit this signal as they
    # catch status messages from the SubChannel by overloading _recv_package().
    @property
    def received(self):
        """Signal that is emitted when new data is received. Multiple
        arrived messages may result in a single call to this method.
        There is no guarantee that recv() has not been called in the
        mean time. The signal is emitted with the channel instance
        as argument.
        """
        return self._received_signal

    ## Properties

    @property
    def pending(self):
        """Get the number of pending incoming messages."""
        return len(self._q_in)

    @property
    def closed(self):
        """Get whether the channel is closed."""
        return self._closed

    @property
    def slot_outgoing(self):
        """Get the outgoing slot name."""
        return self._slot_out

    @property
    def slot_incoming(self):
        """Get the incoming slot name."""
        return self._slot_in
コード例 #4
0
ファイル: channels_base.py プロジェクト: ysalmon/pyzo
class BaseChannel(object):
    """ BaseChannel(context, slot_base, message_type=yoton.TEXT)
    
    Abstract class for all channels.
    
    Parameters
    ----------
    context : yoton.Context instance
        The context that this channel uses to send messages in a network.
    slot_base : string
        The base slot name. The channel appends an extension to indicate
        message type and messaging pattern to create the final slot name.
        The final slot is used to connect channels at different contexts
        in a network
    message_type : yoton.MessageType instance
        (default is yoton.TEXT)
        Object to convert messages to bytes and bytes to messages.
        Users can create their own message_type class to enable
        communicating any type of message they want.
    
    Details
    -------
    Messages send via a channel are delivered asynchronically to the
    corresponding channels.
    
    All channels are associated with a context and can be used to send
    messages to other channels in the network. Each channel is also
    associated with a slot, which is a string that represents a kind
    of address. A message send by a channel at slot X can only be received
    by a channel with slot X.
    
    Note that the channel appends an extension
    to the user-supplied slot name, that represents the message type
    and messaging pattern of the channel. In this way, it is prevented
    that for example a PubChannel can communicate with a RepChannel.
    
    """
    
    def __init__(self, context, slot_base, message_type=None):
        
        # Store context
        if not isinstance(context, Context):
            raise ValueError('Context not valid.')
        self._context = context
        
        # Check message type
        if message_type is None:
            message_type = TEXT
        if isinstance(message_type, type) and issubclass(message_type, MessageType):
            message_type = message_type()
        if isinstance(message_type, MessageType):
            message_type = message_type
        else:
            raise ValueError('message_type should be a MessageType instance.')
        
        # Store message type and conversion methods
        self._message_type_instance = message_type
        self.message_from_bytes = message_type.message_from_bytes
        self.message_to_bytes = message_type.message_to_bytes
        
        # Queue for incoming trafic (not used for pure sending channels)
        self._q_in = PackageQueue(*context._queue_params)
        
        # For sending channels: to lock the channel for sending
        self._send_condition = threading.Condition()
        self._is_send_locked = 0 # "True" is the timeout time
        
        # Signal for receiving data
        self._received_signal = yoton.events.Signal()
        self._posted_received_event = False
        
        # Channels can be closed
        self._closed = False
        
        # Event driven mode
        self._run_mode = 0
        
        # Init slots
        self._init_slots(slot_base)
    
    
    def _init_slots(self, slot_base):
        """ _init_slots(slot_base)
        
        Called from __init__ to initialize the slots and perform all checks.
        
        """
        
        # Check if slot is string
        if not isinstance(slot_base, basestring):
            raise ValueError('slot_base must be a string.')
        
        # Get full slot names, init byte versions
        slots_t = []
        slots_h = []
        
        # Get extension for message type and messaging pattern
        ext_type = self._message_type_instance.message_type_name()
        ext_patterns = self._messaging_patterns() # (incoming, outgoing)
        
        # Normalize and check slot names
        for ext_pattern in ext_patterns:
            if not ext_pattern:
                slots_t.append(None)
                slots_h.append(0)
                continue
            # Get full name
            slot = slot_base + '.' + ext_type + '.' + ext_pattern
            # Store text version
            slots_t.append(slot)
            # Strip and make lowercase
            slot = slot.strip().lower()
            # Hash
            slots_h.append(slot_hash(slot))
        
        # Store slots
        self._slot_out = slots_t[0]
        self._slot_in = slots_t[1]
        self._slot_out_h = slots_h[0]
        self._slot_in_h = slots_h[1]
        
        # Register slots (warn if neither slot is valid)
        if self._slot_out_h:
            self._context._register_sending_channel(self, self._slot_out_h, self._slot_out)
        if self._slot_in_h:
            self._context._register_receiving_channel(self, self._slot_in_h, self._slot_in)
        if not self._slot_out_h and not self._slot_in_h:
            raise ValueError('This channel does not have valid slots.')
    
    
    def _messaging_patterns(self):
        """ _messaging_patterns()
        
        Implement to return a string that specifies the pattern
        for sending and receiving, respecitively.
        
        """
        raise NotImplementedError()
    
    
    def close(self):
        """ close()
        
        Close the channel, i.e. unregisters this channel at the context.
        A closed channel cannot be reused.
        
        Future attempt to send() messages will result in an IOError
        being raised. Messages currently in the channel's queue can
        still be recv()'ed, but no new messages will be delivered at
        this channel.
        
        """
        # We keep a reference to the context, otherwise we need locks
        # The context clears the reference to this channel when unregistering.
        self._closed = True
        self._context._unregister_channel(self)
    
    
    def _send(self, message, dest_id=0, dest_seq=0):
        """ _send(message, dest_id=0, dest_seq=0)
        
        Sends a message of raw bytes without checking whether they're bytes.
        Optionally, dest_id and dest_seq represent the message that
        this message  replies to. These are used for the request/reply
        pattern.
        
        Returns the package that will be send (or None). The context
        will set _source_id on the package right before
        sending it away.
        
        """
        
        # Check if still open
        if self._closed:
            className = self.__class__.__name__
            raise IOError("Cannot send from closed %s %i." % (className, id(self)))
        
        
        if message:
            # If send_locked, wait at most one second
            if self._is_send_locked:
                self._send_condition.acquire()
                try:
                    self._send_condition.wait(1.0) # wait for notify
                finally:
                    self._send_condition.release()
                    if time.time() > self._is_send_locked:
                        self._is_send_locked = 0
            # Push it on the queue as a package
            slot = self._slot_out_h
            cid = self._context._id
            p = Package(message, slot, cid, 0, dest_id, dest_seq, 0)
            self._context._send_package(p)
            # Return package
            return p
        else:
            return None
    
    
    def _recv(self, block):
        """ _recv(block)
        
        Receive a package (or None).
        
        """
    
        if block is True:
            # Block for 0.25 seconds so that KeyboardInterrupt works
            while not self._closed:
                try:
                    return self._q_in.pop(0.25)
                except self._q_in.Empty:
                    continue
        
        else:
            # Block normal
            try:
                return self._q_in.pop(block)
            except self._q_in.Empty:
                return None
    
    
    def _set_send_lock(self, value):
        """ _set_send_lock(self, value)
        
        Set or unset the blocking for the _send() method.
        
        """
        # Set send lock variable. We adopt a timeout (10s) just in case
        # the SubChannel that locks the PubChannel gets disconnected and
        # is unable to unlock it.
        if value:
            self._is_send_locked = time.time() + 10.0
        else:
            self._is_send_locked = 0
        # Notify any threads that are waiting in _send()
        if not value:
            self._send_condition.acquire()
            try:
                self._send_condition.notifyAll()
            finally:
                self._send_condition.release()
    
    
    ## How packages are inserted in this channel for receiving
    
    
    def _inject_package(self, package):
        """ _inject_package(package)
        
        Same as _recv_package, but by definition do not block.
        _recv_package is overloaded in SubChannel. _inject_package is not.
        
        """
        self._q_in.push(package)
        self._maybe_emit_received()
    
    
    def _recv_package(self, package):
        """ _recv_package(package)
        
        Put package in the queue.
        
        """
        self._q_in.push(package)
        self._maybe_emit_received()
    
    
    def _maybe_emit_received(self):
        """ _maybe_emit_received()
        
        We want to emit a signal, but in such a way that multiple
        arriving packages result in a single emit. This methods
        only posts an event if it has not been done, or if the previous
        event has been handled.
        
        """
        if not self._posted_received_event:
            self._posted_received_event = True
            event = yoton.events.Event(self._emit_received)
            yoton.app.post_event(event)
    
    
    def _emit_received(self):
        """ _emit_received()
        
        Emits the "received" signal. This method is called once new data
        has been received. However, multiple arrived messages may
        result in a single call to this method. There is also no
        guarantee that recv() has not been called in the mean time.
        
        Also sets the variabele so that a new event for this may be
        created. This method is called from the event loop.
        
        """
        self._posted_received_event = False # Reset
        self.received.emit_now(self)
    
    
    # Received property sits on the BaseChannel because is is used by almost
    # all channels. Note that PubChannels never emit this signal as they
    # catch status messages from the SubChannel by overloading _recv_package().
    @property
    def received(self):
        """ Signal that is emitted when new data is received. Multiple
        arrived messages may result in a single call to this method.
        There is no guarantee that recv() has not been called in the
        mean time. The signal is emitted with the channel instance
        as argument.
        """
        return self._received_signal
    
    
    ## Properties
    
    
    @property
    def pending(self):
        """ Get the number of pending incoming messages.
        """
        return len(self._q_in)
    
    
    @property
    def closed(self):
        """ Get whether the channel is closed.
        """
        return self._closed
    
    
    @property
    def slot_outgoing(self):
        """ Get the outgoing slot name.
        """
        return self._slot_out
    
    
    @property
    def slot_incoming(self):
        """ Get the incoming slot name.
        """
        return self._slot_in