def __init__(self, verbose=0, queue_params=None):
# Whether or not to write status information
self._verbose = verbose
# Store queue parameters
if queue_params is None:
queue_params = BUF_MAX_LEN, 'old'
if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
raise ValueError('queue_params should be a 2-element tuple.')
self._queue_params = queue_params
# Create unique key to identify this context
self._id = UID().get_int()
# Channels currently registered. Maps slots to channel instance.
self._sending_channels = {}
self._receiving_channels = {}
# The list of connections
self._connections = []
self._connections_lock = threading.RLock()
# Queue used during startup to collect packages
# This queue is also subject to the _connections_lock
self._startupQueue = PackageQueue(*queue_params)
# For numbering and routing the packages
self._send_seq = 0
self._recv_seq = 0
self._source_map = {}
def __init__(self, verbose=0, queue_params=None):
# Whether or not to write status information
self._verbose = verbose
# Store queue parameters
if queue_params is None:
queue_params = BUF_MAX_LEN, 'old'
if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
raise ValueError('queue_params should be a 2-element tuple.')
self._queue_params = queue_params
# Create unique key to identify this context
self._id = UID().get_int()
# Channels currently registered. Maps slots to channel instance.
self._sending_channels = {}
self._receiving_channels = {}
# The list of connections
self._connections = []
self._connections_lock = threading.RLock()
# Queue used during startup to collect packages
# This queue is also subject to the _connections_lock
self._startupQueue = PackageQueue(*queue_params)
# For numbering and routing the packages
self._send_seq = 0
self._recv_seq = 0
self._source_map = {}
def __init__(self, context, slot_base, message_type=None):
# Store context
if not isinstance(context, Context):
raise ValueError("Context not valid.")
self._context = context
# Check message type
if message_type is None:
message_type = TEXT
if isinstance(message_type, type) and issubclass(
message_type, MessageType):
message_type = message_type()
if isinstance(message_type, MessageType):
message_type = message_type
else:
raise ValueError("message_type should be a MessageType instance.")
# Store message type and conversion methods
self._message_type_instance = message_type
self.message_from_bytes = message_type.message_from_bytes
self.message_to_bytes = message_type.message_to_bytes
# Queue for incoming trafic (not used for pure sending channels)
self._q_in = PackageQueue(*context._queue_params)
# For sending channels: to lock the channel for sending
self._send_condition = threading.Condition()
self._is_send_locked = 0 # "True" is the timeout time
# Signal for receiving data
self._received_signal = yoton.events.Signal()
self._posted_received_event = False
# Channels can be closed
self._closed = False
# Event driven mode
self._run_mode = 0
# Init slots
self._init_slots(slot_base)
def __init__(self, context, slot_base, message_type=None):
# Store context
if not isinstance(context, Context):
raise ValueError('Context not valid.')
self._context = context
# Check message type
if message_type is None:
message_type = TEXT
if isinstance(message_type, type) and issubclass(message_type, MessageType):
message_type = message_type()
if isinstance(message_type, MessageType):
message_type = message_type
else:
raise ValueError('message_type should be a MessageType instance.')
# Store message type and conversion methods
self._message_type_instance = message_type
self.message_from_bytes = message_type.message_from_bytes
self.message_to_bytes = message_type.message_to_bytes
# Queue for incoming trafic (not used for pure sending channels)
self._q_in = PackageQueue(*context._queue_params)
# For sending channels: to lock the channel for sending
self._send_condition = threading.Condition()
self._is_send_locked = 0 # "True" is the timeout time
# Signal for receiving data
self._received_signal = yoton.events.Signal()
self._posted_received_event = False
# Channels can be closed
self._closed = False
# Event driven mode
self._run_mode = 0
# Init slots
self._init_slots(slot_base)
class Context(object):
""" Context(verbose=0, queue_params=None)
A context represents a node in the network. It can connect to
multiple other contexts (using a yoton.Connection.
These other contexts can be in
another process on the same machine, or on another machine
connected via a network or the internet.
This class represents a context that can be used by channel instances
to communicate to other channels in the network. (Thus the name.)
The context is the entity that queue routes the packages produced
by the channels to the other context in the network, where
the packages are distributed to the right channels. A context queues
packages while it is not connected to any other context.
If messages are send on a channel registered at this context while
the context is not connected, the messages are stored by the
context and will be send to the first connecting context.
Example 1
---------
# Create context and bind to a port on localhost
context = yoton.Context()
context.bind('localhost:11111')
# Create a channel and send a message
pub = yoton.PubChannel(context, 'test')
pub.send('Hello world!')
Example 2
---------
# Create context and connect to the port on localhost
context = yoton.Context()
context.connect('localhost:11111')
# Create a channel and receive a message
sub = yoton.SubChannel(context, 'test')
print(sub.recv() # Will print 'Hello world!'
Queue params
------------
The queue_params parameter allows one to specify the package queues
used in the system. It is recommended to use the same parameters
for every context in the network. The value of queue_params should
be a 2-element tuple specifying queue size and discard mode. The
latter can be 'old' (default) or 'new', meaning that if the queue
is full, either the oldest or newest messages are discarted.
"""
def __init__(self, verbose=0, queue_params=None):
# Whether or not to write status information
self._verbose = verbose
# Store queue parameters
if queue_params is None:
queue_params = BUF_MAX_LEN, 'old'
if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
raise ValueError('queue_params should be a 2-element tuple.')
self._queue_params = queue_params
# Create unique key to identify this context
self._id = UID().get_int()
# Channels currently registered. Maps slots to channel instance.
self._sending_channels = {}
self._receiving_channels = {}
# The list of connections
self._connections = []
self._connections_lock = threading.RLock()
# Queue used during startup to collect packages
# This queue is also subject to the _connections_lock
self._startupQueue = PackageQueue(*queue_params)
# For numbering and routing the packages
self._send_seq = 0
self._recv_seq = 0
self._source_map = {}
def close(self):
""" close()
Close the context in a nice way, by closing all connections
and all channels.
Closing a connection means disconnecting two contexts. Closing
a channel means disasociating a channel from its context.
Unlike connections and channels, a Context instance can be reused
after closing (although this might not always the best strategy).
"""
# Close all connections (also the waiting connections!)
for c in self.connections_all:
c.close('Closed by the context.')
# Close all channels
self.close_channels()
def close_channels(self):
""" close_channels()
Close all channels associated with this context. This does
not close the connections. See also close().
"""
# Get all channels
channels1 = [c for c in self._sending_channels.values()]
channels2 = [c for c in self._receiving_channels.values()]
# Close all channels
for c in set(channels1 + channels2):
c.close()
## Properties
@property
def connections_all(self):
""" Get a list of all Connection instances currently
associated with this context, including pending connections
(connections waiting for another end to connect).
In addition to normal list indexing, the connections objects can be
queried from this list using their name.
"""
# Lock
self._connections_lock.acquire()
try:
return [c for c in self._connections if c.is_alive]
finally:
self._connections_lock.release()
@property
def connections(self):
""" Get a list of the Connection instances currently
active for this context.
In addition to normal list indexing, the connections objects can be
queried from this list using their name.
"""
# Lock
self._connections_lock.acquire()
try:
# Clean up any dead connections
copy = ConnectionCollection()
to_remove = []
for c in self._connections:
if not c.is_alive:
to_remove.append(c)
elif c.is_connected:
copy.append(c)
# Clean
for c in to_remove:
self._connections.remove(c)
# Return copy
return copy
finally:
self._connections_lock.release()
@property
def connection_count(self):
""" Get the number of connected contexts. Can be used as a boolean
to check if the context is connected to any other context.
"""
return len(self.connections)
@property
def id(self):
""" The 8-byte UID of this context.
"""
return self._id
## Public methods
def bind(self, address, max_tries=1, name=''):
""" bind(address, max_tries=1, name='')
Setup a connection with another Context, by being the host.
This method starts a thread that waits for incoming connections.
Error messages are printed when an attemped connect fails. the
thread keeps trying until a successful connection is made, or until
the connection is closed.
Returns a Connection instance that represents the
connection to the other context. These connection objects
can also be obtained via the Context.connections property.
Parameters
----------
address : str
Should be of the shape hostname:port. The port should be an
integer number between 1024 and 2**16. If port does not
represent a number, a valid port number is created using a
hash function.
max_tries : int
The number of ports to try; starting from the given port,
subsequent ports are tried until a free port is available.
The final port can be obtained using the 'port' property of
the returned Connection instance.
name : string
The name for the created Connection instance. It can
be used as a key in the connections property.
Notes on hostname
-----------------
The hostname can be:
* The IP address, or the string hostname of this computer.
* 'localhost': the connections is only visible from this computer.
Also some low level networking layers are bypassed, which results
in a faster connection. The other context should also connect to
'localhost'.
* 'publichost': the connection is visible by other computers on the
same network. Optionally an integer index can be appended if
the machine has multiple IP addresses (see socket.gethostbyname_ex).
"""
# Trigger cleanup of closed connections
self.connections
# Split address in protocol, real hostname and port number
protocol, hostname, port = split_address(address)
# Based on protocol, instantiate connection class (currently only tcp)
if False: #protocol == 'itc':
connection = ItcConnection(self, name)
else:
connection = TcpConnection(self, name)
# Bind connection
connection._bind(hostname, port, max_tries)
# Save connection instance
self._connections_lock.acquire()
try:
# Push packages from startup queue
while len(self._startupQueue):
connection._inject_package(self._startupQueue.pop())
# Add connection object to list of connections
self._connections.append(connection)
finally:
self._connections_lock.release()
# Return Connection instance
return connection
def connect(self, address, timeout=1.0, name=''):
""" connect(self, address, timeout=1.0, name='')
Setup a connection with another context, by connection to a
hosting context. An error is raised when the connection could
not be made.
Returns a Connection instance that represents the
connection to the other context. These connection objects
can also be obtained via the Context.connections property.
Parameters
----------
address : str
Should be of the shape hostname:port. The port should be an
integer number between 1024 and 2**16. If port does not
represent a number, a valid port number is created using a
hash function.
max_tries : int
The number of ports to try; starting from the given port,
subsequent ports are tried until a free port is available.
The final port can be obtained using the 'port' property of
the returned Connection instance.
name : string
The name for the created Connection instance. It can
be used as a key in the connections property.
Notes on hostname
-----------------
The hostname can be:
* The IP address, or the string hostname of this computer.
* 'localhost': the connection is only visible from this computer.
Also some low level networking layers are bypassed, which results
in a faster connection. The other context should also host as
'localhost'.
* 'publichost': the connection is visible by other computers on the
same network. Optionally an integer index can be appended if
the machine has multiple IP addresses (see socket.gethostbyname_ex).
"""
# Trigger cleanup of closed connections
self.connections
# Split address in protocol, real hostname and port number
protocol, hostname, port = split_address(address)
# Based on protocol, instantiate connection class (currently only tcp)
if False: #protocol == 'itc':
connection = ItcConnection(self, name)
else:
connection = TcpConnection(self, name)
# Create new connection and connect it
connection._connect(hostname, port, timeout)
# Save connection instance
self._connections_lock.acquire()
try:
# Push packages from startup queue
while self._startupQueue:
connection._inject_package(self._startupQueue.pop())
# Add connection object to list of connections
self._connections.append(connection)
finally:
self._connections_lock.release()
# Send message in the network to signal a new connection
bb = 'NEW_CONNECTION'.encode('utf-8')
p = Package(bb, SLOT_CONTEXT, self._id, 0, 0, 0, 0)
self._send_package(p)
# Return Connection instance
return connection
def flush(self, timeout=5.0):
""" flush(timeout=5.0)
Wait until all pending messages are send. This will flush all
messages posted from the calling thread. However, it is not
guaranteed that no new messages are posted from another thread.
Raises an error when the flushing times out.
"""
# Flush all connections
for c in self.connections:
c.flush(timeout)
# Done (backward compatibility)
return True
## Private methods used by the Channel classes
def _register_sending_channel(self, channel, slot, slotname=''):
""" _register_sending_channel(channel, slot, slotname='')
The channel objects use this method to register themselves
at a particular slot.
"""
# Check if this slot is free
if slot in self._sending_channels:
raise ValueError("Slot not free: " + str(slotname))
# Register
self._sending_channels[slot] = channel
def _register_receiving_channel(self, channel, slot, slotname=''):
""" _register_receiving_channel(channel, slot, slotname='')
The channel objects use this method to register themselves
at a particular slot.
"""
# Check if this slot is free
if slot in self._receiving_channels:
raise ValueError("Slot not free: " + str(slotname))
# Register
self._receiving_channels[slot] = channel
def _unregister_channel(self, channel):
""" _unregister_channel(channel)
Unregisters the given channel. That channel can no longer
receive messages, and should no longer send messages.
"""
for D in [self._receiving_channels, self._sending_channels]:
for key in [key for key in D.keys()]:
if D[key] == channel:
D.pop(key)
## Private methods to pass packages between context and io-threads
def _send_package(self, package):
""" _send_package(package)
Used by the channels to send a package into the network.
This method routes the package to all currentlt connected
connections. If there are none, the packages is queued at
the context.
"""
# Add number
self._send_seq += 1
package._source_seq = self._send_seq
# Send to all connections, or queue if there are none
self._connections_lock.acquire()
try:
ok = False
for c in self._connections:
if c.is_alive: # Waiting or connected
c._send_package(package)
ok = True
# Should we queue the package?
if not ok:
self._startupQueue.push(package)
finally:
self._connections_lock.release()
def _recv_package(self, package, connection):
""" _recv_package(package, connection)
Used by the connections to receive a package at this
context. The package is distributed to all connections
except the calling one. The package is also distributed
to the right channel (if applicable).
"""
# Get slot
slot = package._slot
# Init what to do with the package
send_further = False
deposit_here = False
# Get what to do with the package
last_seq = self._source_map.get(package._source_id, 0)
if last_seq < package._source_seq:
# Update source map
self._source_map[package._source_id] = package._source_seq
if package._dest_id == 0:
# Add to both lists, first attach seq nr
self._recv_seq += 1
package._recv_seq = self._recv_seq
send_further, deposit_here = True, True
elif package._dest_id == self._id:
# Add only to process list, first attach seq nr
self._recv_seq += 1
package._recv_seq = self._recv_seq
deposit_here = True
else:
# Send package to connected nodes
send_further = True
# Send package to other context (over all alive connections)
if send_further:
self._connections_lock.acquire()
try:
for c in self._connections:
if c is connection or not c.is_alive:
continue
c._send_package(package)
finally:
self._connections_lock.release()
# Process package here or pass to channel
if deposit_here:
if slot == SLOT_CONTEXT:
# Context-to-context messaging;
# A slot starting with a space reprsents the context
self._recv_context_package(package)
else:
# Give package to a channel (if applicable)
channel = self._receiving_channels.get(slot, None)
if channel is not None:
channel._recv_package(package)
def _recv_context_package(self, package):
""" _recv_context_package(package)
Process a package addressed at the context itself. This is how
the context handles higher-level connection tasks.
"""
# Get message: context messages are always utf-8 encoded strings
message = package._data.decode('utf-8')
if message == 'CLOSE_CONNECTION':
# Close the connection. Check which one of our connections is
# connected with the context that send this message.
self._connections_lock.acquire()
try:
for c in self.connections:
if c.is_connected and c.id2 == package._source_id:
c.close(connection.STOP_CLOSED_FROM_THERE, False)
finally:
self._connections_lock.release()
elif message == 'NEW_CONNECTION':
# Resend all status channels
for channel in self._sending_channels.values():
if hasattr(channel, '_current_message') and hasattr(
channel, 'send_last'):
channel.send_last()
else:
print('Yoton: Received unknown context message: ' + message)
class BaseChannel(object):
"""BaseChannel(context, slot_base, message_type=yoton.TEXT)
Abstract class for all channels.
Parameters
----------
context : yoton.Context instance
The context that this channel uses to send messages in a network.
slot_base : string
The base slot name. The channel appends an extension to indicate
message type and messaging pattern to create the final slot name.
The final slot is used to connect channels at different contexts
in a network
message_type : yoton.MessageType instance
(default is yoton.TEXT)
Object to convert messages to bytes and bytes to messages.
Users can create their own message_type class to enable
communicating any type of message they want.
Details
-------
Messages send via a channel are delivered asynchronically to the
corresponding channels.
All channels are associated with a context and can be used to send
messages to other channels in the network. Each channel is also
associated with a slot, which is a string that represents a kind
of address. A message send by a channel at slot X can only be received
by a channel with slot X.
Note that the channel appends an extension
to the user-supplied slot name, that represents the message type
and messaging pattern of the channel. In this way, it is prevented
that for example a PubChannel can communicate with a RepChannel.
"""
def __init__(self, context, slot_base, message_type=None):
# Store context
if not isinstance(context, Context):
raise ValueError("Context not valid.")
self._context = context
# Check message type
if message_type is None:
message_type = TEXT
if isinstance(message_type, type) and issubclass(
message_type, MessageType):
message_type = message_type()
if isinstance(message_type, MessageType):
message_type = message_type
else:
raise ValueError("message_type should be a MessageType instance.")
# Store message type and conversion methods
self._message_type_instance = message_type
self.message_from_bytes = message_type.message_from_bytes
self.message_to_bytes = message_type.message_to_bytes
# Queue for incoming trafic (not used for pure sending channels)
self._q_in = PackageQueue(*context._queue_params)
# For sending channels: to lock the channel for sending
self._send_condition = threading.Condition()
self._is_send_locked = 0 # "True" is the timeout time
# Signal for receiving data
self._received_signal = yoton.events.Signal()
self._posted_received_event = False
# Channels can be closed
self._closed = False
# Event driven mode
self._run_mode = 0
# Init slots
self._init_slots(slot_base)
def _init_slots(self, slot_base):
"""_init_slots(slot_base)
Called from __init__ to initialize the slots and perform all checks.
"""
# Check if slot is string
if not isinstance(slot_base, basestring):
raise ValueError("slot_base must be a string.")
# Get full slot names, init byte versions
slots_t = []
slots_h = []
# Get extension for message type and messaging pattern
ext_type = self._message_type_instance.message_type_name()
ext_patterns = self._messaging_patterns() # (incoming, outgoing)
# Normalize and check slot names
for ext_pattern in ext_patterns:
if not ext_pattern:
slots_t.append(None)
slots_h.append(0)
continue
# Get full name
slot = slot_base + "." + ext_type + "." + ext_pattern
# Store text version
slots_t.append(slot)
# Strip and make lowercase
slot = slot.strip().lower()
# Hash
slots_h.append(slot_hash(slot))
# Store slots
self._slot_out = slots_t[0]
self._slot_in = slots_t[1]
self._slot_out_h = slots_h[0]
self._slot_in_h = slots_h[1]
# Register slots (warn if neither slot is valid)
if self._slot_out_h:
self._context._register_sending_channel(self, self._slot_out_h,
self._slot_out)
if self._slot_in_h:
self._context._register_receiving_channel(self, self._slot_in_h,
self._slot_in)
if not self._slot_out_h and not self._slot_in_h:
raise ValueError("This channel does not have valid slots.")
def _messaging_patterns(self):
"""_messaging_patterns()
Implement to return a string that specifies the pattern
for sending and receiving, respecitively.
"""
raise NotImplementedError()
def close(self):
"""close()
Close the channel, i.e. unregisters this channel at the context.
A closed channel cannot be reused.
Future attempt to send() messages will result in an IOError
being raised. Messages currently in the channel's queue can
still be recv()'ed, but no new messages will be delivered at
this channel.
"""
# We keep a reference to the context, otherwise we need locks
# The context clears the reference to this channel when unregistering.
self._closed = True
self._context._unregister_channel(self)
def _send(self, message, dest_id=0, dest_seq=0):
"""_send(message, dest_id=0, dest_seq=0)
Sends a message of raw bytes without checking whether they're bytes.
Optionally, dest_id and dest_seq represent the message that
this message replies to. These are used for the request/reply
pattern.
Returns the package that will be send (or None). The context
will set _source_id on the package right before
sending it away.
"""
# Check if still open
if self._closed:
className = self.__class__.__name__
raise IOError("Cannot send from closed %s %i." %
(className, id(self)))
if message:
# If send_locked, wait at most one second
if self._is_send_locked:
self._send_condition.acquire()
try:
self._send_condition.wait(1.0) # wait for notify
finally:
self._send_condition.release()
if time.time() > self._is_send_locked:
self._is_send_locked = 0
# Push it on the queue as a package
slot = self._slot_out_h
cid = self._context._id
p = Package(message, slot, cid, 0, dest_id, dest_seq, 0)
self._context._send_package(p)
# Return package
return p
else:
return None
def _recv(self, block):
"""_recv(block)
Receive a package (or None).
"""
if block is True:
# Block for 0.25 seconds so that KeyboardInterrupt works
while not self._closed:
try:
return self._q_in.pop(0.25)
except self._q_in.Empty:
continue
else:
# Block normal
try:
return self._q_in.pop(block)
except self._q_in.Empty:
return None
def _set_send_lock(self, value):
"""_set_send_lock(self, value)
Set or unset the blocking for the _send() method.
"""
# Set send lock variable. We adopt a timeout (10s) just in case
# the SubChannel that locks the PubChannel gets disconnected and
# is unable to unlock it.
if value:
self._is_send_locked = time.time() + 10.0
else:
self._is_send_locked = 0
# Notify any threads that are waiting in _send()
if not value:
self._send_condition.acquire()
try:
self._send_condition.notifyAll()
finally:
self._send_condition.release()
## How packages are inserted in this channel for receiving
def _inject_package(self, package):
"""_inject_package(package)
Same as _recv_package, but by definition do not block.
_recv_package is overloaded in SubChannel. _inject_package is not.
"""
self._q_in.push(package)
self._maybe_emit_received()
def _recv_package(self, package):
"""_recv_package(package)
Put package in the queue.
"""
self._q_in.push(package)
self._maybe_emit_received()
def _maybe_emit_received(self):
"""_maybe_emit_received()
We want to emit a signal, but in such a way that multiple
arriving packages result in a single emit. This methods
only posts an event if it has not been done, or if the previous
event has been handled.
"""
if not self._posted_received_event:
self._posted_received_event = True
event = yoton.events.Event(self._emit_received)
yoton.app.post_event(event)
def _emit_received(self):
"""_emit_received()
Emits the "received" signal. This method is called once new data
has been received. However, multiple arrived messages may
result in a single call to this method. There is also no
guarantee that recv() has not been called in the mean time.
Also sets the variabele so that a new event for this may be
created. This method is called from the event loop.
"""
self._posted_received_event = False # Reset
self.received.emit_now(self)
# Received property sits on the BaseChannel because is is used by almost
# all channels. Note that PubChannels never emit this signal as they
# catch status messages from the SubChannel by overloading _recv_package().
@property
def received(self):
"""Signal that is emitted when new data is received. Multiple
arrived messages may result in a single call to this method.
There is no guarantee that recv() has not been called in the
mean time. The signal is emitted with the channel instance
as argument.
"""
return self._received_signal
## Properties
@property
def pending(self):
"""Get the number of pending incoming messages."""
return len(self._q_in)
@property
def closed(self):
"""Get whether the channel is closed."""
return self._closed
@property
def slot_outgoing(self):
"""Get the outgoing slot name."""
return self._slot_out
@property
def slot_incoming(self):
"""Get the incoming slot name."""
return self._slot_in
class YotonApplication(object):
""" YotonApplication
Represents the yoton application and contains functions for
the event system. Multiple instances can be created, they will
all operate on the same event queue and share attributes
(because these are on the class, not on the instance).
One instance of this class is always accesible via yoton.app.
For convenience, several of its methods are also accessible
directly from the yoton module namespace.
"""
# Event queues
_event_queue = PackageQueue(10000, 'new')
# Flag to stop event loop
_stop_event_loop = False
# Flag to signal whether we are in an event loop
# Can be set externally if the event loop is hijacked.
_in_event_loop = False
# To allow other event loops to embed the yoton event loop
_embedding_callback1 = None # The reference
_embedding_callback2 = None # Used in post_event
def call_later(self, func, timeout=0.0, *args, **kwargs):
""" call_later(func, timeout=0.0, *args, **kwargs)
Call the given function after the specified timeout.
Parameters
----------
func : callable
The function to call.
timeout : number
The time to wait in seconds. If zero, the event is put on the event
queue. If negative, the event will be put at the front of the event
queue, so that it's processed asap.
args : arguments
The arguments to call func with.
kwargs: keyword arguments.
The keyword arguments to call func with.
"""
# Wrap the object in an event
event = Event(func, *args, **kwargs)
# Put it in the queue
if timeout > 0:
self.post_event_later(event, timeout)
elif timeout < 0:
self.post_event_asap(event) # priority event
else:
self.post_event(event)
def post_event(self, event):
""" post_event(events)
Post an event to the event queue.
"""
YotonApplication._event_queue.push(event)
#
if YotonApplication._embedding_callback2 is not None:
YotonApplication._embedding_callback2 = None
YotonApplication._embedding_callback1()
def post_event_asap(self, event):
""" post_event_asap(event)
Post an event to the event queue. Handle as soon as possible;
putting it in front of the queue.
"""
YotonApplication._event_queue.insert(event)
#
if YotonApplication._embedding_callback2 is not None:
YotonApplication._embedding_callback2 = None
YotonApplication._embedding_callback1()
def post_event_later(self, event, delay):
""" post_event_later(event, delay)
Post an event to the event queue, but with a certain delay.
"""
event._timeout = time.time() + delay
theTimerThread.add(event)
# Calls post_event in due time
def process_events(self, block=False):
""" process_events(block=False)
Process all yoton events currently in the queue.
This function should be called periodically
in order to keep the yoton event system running.
block can be False (no blocking), True (block), or a float
blocking for maximally 'block' seconds.
"""
# Reset callback for the embedding event loop
YotonApplication._embedding_callback2 = YotonApplication._embedding_callback1
# Process events
try:
while True:
event = YotonApplication._event_queue.pop(block)
event.dispatch()
block = False # Proceed until there are now more events
except PackageQueue.Empty:
pass
def start_event_loop(self):
""" start_event_loop()
Enter an event loop that keeps calling yoton.process_events().
The event loop can be stopped using stop_event_loop().
"""
# Dont go if we are in an event loop
if YotonApplication._in_event_loop:
return
# Set flags
YotonApplication._stop_event_loop = False
YotonApplication._in_event_loop = True
try:
# Keep blocking for 3 seconds so a keyboardinterrupt still works
while not YotonApplication._stop_event_loop:
self.process_events(3.0)
finally:
# Unset flag
YotonApplication._in_event_loop = False
def stop_event_loop(self):
""" stop_event_loop()
Stops the event loop if it is running.
"""
if not YotonApplication._stop_event_loop:
# Signal stop
YotonApplication._stop_event_loop = True
# Push an event so that process_events() unblocks
def dummy():
pass
self.post_event(Event(dummy))
def embed_event_loop(self, callback):
""" embed_event_loop(callback)
Embed the yoton event loop in another event loop. The given callback
is called whenever a new yoton event is created. The callback
should create an event in the other event-loop, which should
lead to a call to the process_events() method. The given callback
should be thread safe.
Use None as an argument to disable the embedding.
"""
YotonApplication._embedding_callback1 = callback
YotonApplication._embedding_callback2 = callback
class Context(object):
""" Context(verbose=0, queue_params=None)
A context represents a node in the network. It can connect to
multiple other contexts (using a yoton.Connection.
These other contexts can be in
another process on the same machine, or on another machine
connected via a network or the internet.
This class represents a context that can be used by channel instances
to communicate to other channels in the network. (Thus the name.)
The context is the entity that queue routes the packages produced
by the channels to the other context in the network, where
the packages are distributed to the right channels. A context queues
packages while it is not connected to any other context.
If messages are send on a channel registered at this context while
the context is not connected, the messages are stored by the
context and will be send to the first connecting context.
Example 1
---------
# Create context and bind to a port on localhost
context = yoton.Context()
context.bind('localhost:11111')
# Create a channel and send a message
pub = yoton.PubChannel(context, 'test')
pub.send('Hello world!')
Example 2
---------
# Create context and connect to the port on localhost
context = yoton.Context()
context.connect('localhost:11111')
# Create a channel and receive a message
sub = yoton.SubChannel(context, 'test')
print(sub.recv() # Will print 'Hello world!'
Queue params
------------
The queue_params parameter allows one to specify the package queues
used in the system. It is recommended to use the same parameters
for every context in the network. The value of queue_params should
be a 2-element tuple specifying queue size and discard mode. The
latter can be 'old' (default) or 'new', meaning that if the queue
is full, either the oldest or newest messages are discarted.
"""
def __init__(self, verbose=0, queue_params=None):
# Whether or not to write status information
self._verbose = verbose
# Store queue parameters
if queue_params is None:
queue_params = BUF_MAX_LEN, 'old'
if not (isinstance(queue_params, tuple) and len(queue_params) == 2):
raise ValueError('queue_params should be a 2-element tuple.')
self._queue_params = queue_params
# Create unique key to identify this context
self._id = UID().get_int()
# Channels currently registered. Maps slots to channel instance.
self._sending_channels = {}
self._receiving_channels = {}
# The list of connections
self._connections = []
self._connections_lock = threading.RLock()
# Queue used during startup to collect packages
# This queue is also subject to the _connections_lock
self._startupQueue = PackageQueue(*queue_params)
# For numbering and routing the packages
self._send_seq = 0
self._recv_seq = 0
self._source_map = {}
def close(self):
""" close()
Close the context in a nice way, by closing all connections
and all channels.
Closing a connection means disconnecting two contexts. Closing
a channel means disasociating a channel from its context.
Unlike connections and channels, a Context instance can be reused
after closing (although this might not always the best strategy).
"""
# Close all connections (also the waiting connections!)
for c in self.connections_all:
c.close('Closed by the context.')
# Close all channels
self.close_channels()
def close_channels(self):
""" close_channels()
Close all channels associated with this context. This does
not close the connections. See also close().
"""
# Get all channels
channels1 = [c for c in self._sending_channels.values()]
channels2 = [c for c in self._receiving_channels.values()]
# Close all channels
for c in set(channels1+channels2):
c.close()
## Properties
@property
def connections_all(self):
""" Get a list of all Connection instances currently
associated with this context, including pending connections
(connections waiting for another end to connect).
In addition to normal list indexing, the connections objects can be
queried from this list using their name.
"""
# Lock
self._connections_lock.acquire()
try:
return [c for c in self._connections if c.is_alive]
finally:
self._connections_lock.release()
@property
def connections(self):
""" Get a list of the Connection instances currently
active for this context.
In addition to normal list indexing, the connections objects can be
queried from this list using their name.
"""
# Lock
self._connections_lock.acquire()
try:
# Clean up any dead connections
copy = ConnectionCollection()
to_remove = []
for c in self._connections:
if not c.is_alive:
to_remove.append(c)
elif c.is_connected:
copy.append(c)
# Clean
for c in to_remove:
self._connections.remove(c)
# Return copy
return copy
finally:
self._connections_lock.release()
@property
def connection_count(self):
""" Get the number of connected contexts. Can be used as a boolean
to check if the context is connected to any other context.
"""
return len(self.connections)
@property
def id(self):
""" The 8-byte UID of this context.
"""
return self._id
## Public methods
def bind(self, address, max_tries=1, name=''):
""" bind(address, max_tries=1, name='')
Setup a connection with another Context, by being the host.
This method starts a thread that waits for incoming connections.
Error messages are printed when an attemped connect fails. the
thread keeps trying until a successful connection is made, or until
the connection is closed.
Returns a Connection instance that represents the
connection to the other context. These connection objects
can also be obtained via the Context.connections property.
Parameters
----------
address : str
Should be of the shape hostname:port. The port should be an
integer number between 1024 and 2**16. If port does not
represent a number, a valid port number is created using a
hash function.
max_tries : int
The number of ports to try; starting from the given port,
subsequent ports are tried until a free port is available.
The final port can be obtained using the 'port' property of
the returned Connection instance.
name : string
The name for the created Connection instance. It can
be used as a key in the connections property.
Notes on hostname
-----------------
The hostname can be:
* The IP address, or the string hostname of this computer.
* 'localhost': the connections is only visible from this computer.
Also some low level networking layers are bypassed, which results
in a faster connection. The other context should also connect to
'localhost'.
* 'publichost': the connection is visible by other computers on the
same network. Optionally an integer index can be appended if
the machine has multiple IP addresses (see socket.gethostbyname_ex).
"""
# Trigger cleanup of closed connections
self.connections
# Split address in protocol, real hostname and port number
protocol, hostname, port = split_address(address)
# Based on protocol, instantiate connection class (currently only tcp)
if False:#protocol == 'itc':
connection = ItcConnection(self, name)
else:
connection = TcpConnection(self, name)
# Bind connection
connection._bind(hostname, port, max_tries)
# Save connection instance
self._connections_lock.acquire()
try:
# Push packages from startup queue
while len(self._startupQueue):
connection._inject_package(self._startupQueue.pop())
# Add connection object to list of connections
self._connections.append(connection)
finally:
self._connections_lock.release()
# Return Connection instance
return connection
def connect(self, address, timeout=1.0, name=''):
""" connect(self, address, timeout=1.0, name='')
Setup a connection with another context, by connection to a
hosting context. An error is raised when the connection could
not be made.
Returns a Connection instance that represents the
connection to the other context. These connection objects
can also be obtained via the Context.connections property.
Parameters
----------
address : str
Should be of the shape hostname:port. The port should be an
integer number between 1024 and 2**16. If port does not
represent a number, a valid port number is created using a
hash function.
max_tries : int
The number of ports to try; starting from the given port,
subsequent ports are tried until a free port is available.
The final port can be obtained using the 'port' property of
the returned Connection instance.
name : string
The name for the created Connection instance. It can
be used as a key in the connections property.
Notes on hostname
-----------------
The hostname can be:
* The IP address, or the string hostname of this computer.
* 'localhost': the connection is only visible from this computer.
Also some low level networking layers are bypassed, which results
in a faster connection. The other context should also host as
'localhost'.
* 'publichost': the connection is visible by other computers on the
same network. Optionally an integer index can be appended if
the machine has multiple IP addresses (see socket.gethostbyname_ex).
"""
# Trigger cleanup of closed connections
self.connections
# Split address in protocol, real hostname and port number
protocol, hostname, port = split_address(address)
# Based on protocol, instantiate connection class (currently only tcp)
if False:#protocol == 'itc':
connection = ItcConnection(self, name)
else:
connection = TcpConnection(self, name)
# Create new connection and connect it
connection._connect(hostname, port, timeout)
# Save connection instance
self._connections_lock.acquire()
try:
# Push packages from startup queue
while self._startupQueue:
connection._inject_package(self._startupQueue.pop())
# Add connection object to list of connections
self._connections.append(connection)
finally:
self._connections_lock.release()
# Send message in the network to signal a new connection
bb = 'NEW_CONNECTION'.encode('utf-8')
p = Package(bb, SLOT_CONTEXT, self._id, 0,0,0,0)
self._send_package(p)
# Return Connection instance
return connection
def flush(self, timeout=5.0):
""" flush(timeout=5.0)
Wait until all pending messages are send. This will flush all
messages posted from the calling thread. However, it is not
guaranteed that no new messages are posted from another thread.
Raises an error when the flushing times out.
"""
# Flush all connections
for c in self.connections:
c.flush(timeout)
# Done (backward compatibility)
return True
## Private methods used by the Channel classes
def _register_sending_channel(self, channel, slot, slotname=''):
""" _register_sending_channel(channel, slot, slotname='')
The channel objects use this method to register themselves
at a particular slot.
"""
# Check if this slot is free
if slot in self._sending_channels:
raise ValueError("Slot not free: " + str(slotname))
# Register
self._sending_channels[slot] = channel
def _register_receiving_channel(self, channel, slot, slotname=''):
""" _register_receiving_channel(channel, slot, slotname='')
The channel objects use this method to register themselves
at a particular slot.
"""
# Check if this slot is free
if slot in self._receiving_channels:
raise ValueError("Slot not free: " + str(slotname))
# Register
self._receiving_channels[slot] = channel
def _unregister_channel(self, channel):
""" _unregister_channel(channel)
Unregisters the given channel. That channel can no longer
receive messages, and should no longer send messages.
"""
for D in [self._receiving_channels, self._sending_channels]:
for key in [key for key in D.keys()]:
if D[key] == channel:
D.pop(key)
## Private methods to pass packages between context and io-threads
def _send_package(self, package):
""" _send_package(package)
Used by the channels to send a package into the network.
This method routes the package to all currentlt connected
connections. If there are none, the packages is queued at
the context.
"""
# Add number
self._send_seq += 1
package._source_seq = self._send_seq
# Send to all connections, or queue if there are none
self._connections_lock.acquire()
try:
ok = False
for c in self._connections:
if c.is_alive: # Waiting or connected
c._send_package(package)
ok = True
# Should we queue the package?
if not ok:
self._startupQueue.push(package)
finally:
self._connections_lock.release()
def _recv_package(self, package, connection):
""" _recv_package(package, connection)
Used by the connections to receive a package at this
context. The package is distributed to all connections
except the calling one. The package is also distributed
to the right channel (if applicable).
"""
# Get slot
slot = package._slot
# Init what to do with the package
send_further = False
deposit_here = False
# Get what to do with the package
last_seq = self._source_map.get(package._source_id, 0)
if last_seq < package._source_seq:
# Update source map
self._source_map[package._source_id] = package._source_seq
if package._dest_id == 0:
# Add to both lists, first attach seq nr
self._recv_seq += 1
package._recv_seq = self._recv_seq
send_further, deposit_here = True, True
elif package._dest_id == self._id:
# Add only to process list, first attach seq nr
self._recv_seq += 1
package._recv_seq = self._recv_seq
deposit_here = True
else:
# Send package to connected nodes
send_further = True
# Send package to other context (over all alive connections)
if send_further:
self._connections_lock.acquire()
try:
for c in self._connections:
if c is connection or not c.is_alive:
continue
c._send_package(package)
finally:
self._connections_lock.release()
# Process package here or pass to channel
if deposit_here:
if slot == SLOT_CONTEXT:
# Context-to-context messaging;
# A slot starting with a space reprsents the context
self._recv_context_package(package)
else:
# Give package to a channel (if applicable)
channel = self._receiving_channels.get(slot, None)
if channel is not None:
channel._recv_package(package)
def _recv_context_package(self, package):
""" _recv_context_package(package)
Process a package addressed at the context itself. This is how
the context handles higher-level connection tasks.
"""
# Get message: context messages are always utf-8 encoded strings
message = package._data.decode('utf-8')
if message == 'CLOSE_CONNECTION':
# Close the connection. Check which one of our connections is
# connected with the context that send this message.
self._connections_lock.acquire()
try:
for c in self.connections:
if c.is_connected and c.id2 == package._source_id:
c.close(connection.STOP_CLOSED_FROM_THERE, False)
finally:
self._connections_lock.release()
elif message == 'NEW_CONNECTION':
# Resend all status channels
for channel in self._sending_channels.values():
if hasattr(channel, '_current_message') and hasattr(channel, 'send_last'):
channel.send_last()
else:
print('Yoton: Received unknown context message: '+message)
class BaseChannel(object):
""" BaseChannel(context, slot_base, message_type=yoton.TEXT)
Abstract class for all channels.
Parameters
----------
context : yoton.Context instance
The context that this channel uses to send messages in a network.
slot_base : string
The base slot name. The channel appends an extension to indicate
message type and messaging pattern to create the final slot name.
The final slot is used to connect channels at different contexts
in a network
message_type : yoton.MessageType instance
(default is yoton.TEXT)
Object to convert messages to bytes and bytes to messages.
Users can create their own message_type class to enable
communicating any type of message they want.
Details
-------
Messages send via a channel are delivered asynchronically to the
corresponding channels.
All channels are associated with a context and can be used to send
messages to other channels in the network. Each channel is also
associated with a slot, which is a string that represents a kind
of address. A message send by a channel at slot X can only be received
by a channel with slot X.
Note that the channel appends an extension
to the user-supplied slot name, that represents the message type
and messaging pattern of the channel. In this way, it is prevented
that for example a PubChannel can communicate with a RepChannel.
"""
def __init__(self, context, slot_base, message_type=None):
# Store context
if not isinstance(context, Context):
raise ValueError('Context not valid.')
self._context = context
# Check message type
if message_type is None:
message_type = TEXT
if isinstance(message_type, type) and issubclass(message_type, MessageType):
message_type = message_type()
if isinstance(message_type, MessageType):
message_type = message_type
else:
raise ValueError('message_type should be a MessageType instance.')
# Store message type and conversion methods
self._message_type_instance = message_type
self.message_from_bytes = message_type.message_from_bytes
self.message_to_bytes = message_type.message_to_bytes
# Queue for incoming trafic (not used for pure sending channels)
self._q_in = PackageQueue(*context._queue_params)
# For sending channels: to lock the channel for sending
self._send_condition = threading.Condition()
self._is_send_locked = 0 # "True" is the timeout time
# Signal for receiving data
self._received_signal = yoton.events.Signal()
self._posted_received_event = False
# Channels can be closed
self._closed = False
# Event driven mode
self._run_mode = 0
# Init slots
self._init_slots(slot_base)
def _init_slots(self, slot_base):
""" _init_slots(slot_base)
Called from __init__ to initialize the slots and perform all checks.
"""
# Check if slot is string
if not isinstance(slot_base, basestring):
raise ValueError('slot_base must be a string.')
# Get full slot names, init byte versions
slots_t = []
slots_h = []
# Get extension for message type and messaging pattern
ext_type = self._message_type_instance.message_type_name()
ext_patterns = self._messaging_patterns() # (incoming, outgoing)
# Normalize and check slot names
for ext_pattern in ext_patterns:
if not ext_pattern:
slots_t.append(None)
slots_h.append(0)
continue
# Get full name
slot = slot_base + '.' + ext_type + '.' + ext_pattern
# Store text version
slots_t.append(slot)
# Strip and make lowercase
slot = slot.strip().lower()
# Hash
slots_h.append(slot_hash(slot))
# Store slots
self._slot_out = slots_t[0]
self._slot_in = slots_t[1]
self._slot_out_h = slots_h[0]
self._slot_in_h = slots_h[1]
# Register slots (warn if neither slot is valid)
if self._slot_out_h:
self._context._register_sending_channel(self, self._slot_out_h, self._slot_out)
if self._slot_in_h:
self._context._register_receiving_channel(self, self._slot_in_h, self._slot_in)
if not self._slot_out_h and not self._slot_in_h:
raise ValueError('This channel does not have valid slots.')
def _messaging_patterns(self):
""" _messaging_patterns()
Implement to return a string that specifies the pattern
for sending and receiving, respecitively.
"""
raise NotImplementedError()
def close(self):
""" close()
Close the channel, i.e. unregisters this channel at the context.
A closed channel cannot be reused.
Future attempt to send() messages will result in an IOError
being raised. Messages currently in the channel's queue can
still be recv()'ed, but no new messages will be delivered at
this channel.
"""
# We keep a reference to the context, otherwise we need locks
# The context clears the reference to this channel when unregistering.
self._closed = True
self._context._unregister_channel(self)
def _send(self, message, dest_id=0, dest_seq=0):
""" _send(message, dest_id=0, dest_seq=0)
Sends a message of raw bytes without checking whether they're bytes.
Optionally, dest_id and dest_seq represent the message that
this message replies to. These are used for the request/reply
pattern.
Returns the package that will be send (or None). The context
will set _source_id on the package right before
sending it away.
"""
# Check if still open
if self._closed:
className = self.__class__.__name__
raise IOError("Cannot send from closed %s %i." % (className, id(self)))
if message:
# If send_locked, wait at most one second
if self._is_send_locked:
self._send_condition.acquire()
try:
self._send_condition.wait(1.0) # wait for notify
finally:
self._send_condition.release()
if time.time() > self._is_send_locked:
self._is_send_locked = 0
# Push it on the queue as a package
slot = self._slot_out_h
cid = self._context._id
p = Package(message, slot, cid, 0, dest_id, dest_seq, 0)
self._context._send_package(p)
# Return package
return p
else:
return None
def _recv(self, block):
""" _recv(block)
Receive a package (or None).
"""
if block is True:
# Block for 0.25 seconds so that KeyboardInterrupt works
while not self._closed:
try:
return self._q_in.pop(0.25)
except self._q_in.Empty:
continue
else:
# Block normal
try:
return self._q_in.pop(block)
except self._q_in.Empty:
return None
def _set_send_lock(self, value):
""" _set_send_lock(self, value)
Set or unset the blocking for the _send() method.
"""
# Set send lock variable. We adopt a timeout (10s) just in case
# the SubChannel that locks the PubChannel gets disconnected and
# is unable to unlock it.
if value:
self._is_send_locked = time.time() + 10.0
else:
self._is_send_locked = 0
# Notify any threads that are waiting in _send()
if not value:
self._send_condition.acquire()
try:
self._send_condition.notifyAll()
finally:
self._send_condition.release()
## How packages are inserted in this channel for receiving
def _inject_package(self, package):
""" _inject_package(package)
Same as _recv_package, but by definition do not block.
_recv_package is overloaded in SubChannel. _inject_package is not.
"""
self._q_in.push(package)
self._maybe_emit_received()
def _recv_package(self, package):
""" _recv_package(package)
Put package in the queue.
"""
self._q_in.push(package)
self._maybe_emit_received()
def _maybe_emit_received(self):
""" _maybe_emit_received()
We want to emit a signal, but in such a way that multiple
arriving packages result in a single emit. This methods
only posts an event if it has not been done, or if the previous
event has been handled.
"""
if not self._posted_received_event:
self._posted_received_event = True
event = yoton.events.Event(self._emit_received)
yoton.app.post_event(event)
def _emit_received(self):
""" _emit_received()
Emits the "received" signal. This method is called once new data
has been received. However, multiple arrived messages may
result in a single call to this method. There is also no
guarantee that recv() has not been called in the mean time.
Also sets the variabele so that a new event for this may be
created. This method is called from the event loop.
"""
self._posted_received_event = False # Reset
self.received.emit_now(self)
# Received property sits on the BaseChannel because is is used by almost
# all channels. Note that PubChannels never emit this signal as they
# catch status messages from the SubChannel by overloading _recv_package().
@property
def received(self):
""" Signal that is emitted when new data is received. Multiple
arrived messages may result in a single call to this method.
There is no guarantee that recv() has not been called in the
mean time. The signal is emitted with the channel instance
as argument.
"""
return self._received_signal
## Properties
@property
def pending(self):
""" Get the number of pending incoming messages.
"""
return len(self._q_in)
@property
def closed(self):
""" Get whether the channel is closed.
"""
return self._closed
@property
def slot_outgoing(self):
""" Get the outgoing slot name.
"""
return self._slot_out
@property
def slot_incoming(self):
""" Get the incoming slot name.
"""
return self._slot_in
