def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True):
if not isinstance(pat, Pattern):
raise ValueError('pat is not a Pattern instance')
if id_0 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_0)
if id_1 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_1)
if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids):
raise ValueError('unrecognized pattern interface identifiers')
self.log_info('connecting modules {0} and {1}'.format(id_0, id_1))
# Check compatibility of the interfaces exposed by the modules and the
# pattern; since the manager only contains module classes and not class
# instances, we need to create Interface instances from the selectors
# associated with the modules in order to test their compatibility:
if compat_check:
rank_0 = self.rank_to_id.inv[id_0]
rank_1 = self.rank_to_id.inv[id_1]
self.log_info('checking compatibility of modules {0} and {1} and'
' assigned pattern'.format(id_0, id_1))
mod_int_0 = Interface(self._kwargs[rank_0]['sel'])
mod_int_0[self._kwargs[rank_0]['sel']] = 0
mod_int_1 = Interface(self._kwargs[rank_1]['sel'])
mod_int_1[self._kwargs[rank_1]['sel']] = 0
mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in'
mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out'
mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot'
mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike'
mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in'
mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out'
mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot'
mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike'
if not mod_int_0.is_compatible(0, pat.interface, int_0, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_0, int_0))
if not mod_int_1.is_compatible(0, pat.interface, int_1, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_1, int_1))
# XXX Need to check for fan-in XXX
# Store the pattern information in the routing table:
self.log_info('updating routing table with pattern')
if pat.is_connected(0, 1):
self.routing_table[id_0, id_1] = {
'pattern': pat,
'int_0': int_0,
'int_1': int_1
}
if pat.is_connected(1, 0):
self.routing_table[id_1, id_0] = {
'pattern': pat,
'int_0': int_1,
'int_1': int_0
}
self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True):
if not isinstance(pat, Pattern):
raise ValueError('pat is not a Pattern instance')
if id_0 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_0)
if id_1 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_1)
if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids):
raise ValueError('unrecognized pattern interface identifiers')
self.log_info('connecting modules {0} and {1}'
.format(id_0, id_1))
# Check compatibility of the interfaces exposed by the modules and the
# pattern; since the manager only contains module classes and not class
# instances, we need to create Interface instances from the selectors
# associated with the modules in order to test their compatibility:
if compat_check:
rank_0 = self.rank_to_id.inv[id_0]
rank_1 = self.rank_to_id.inv[id_1]
self.log_info('checking compatibility of modules {0} and {1} and'
' assigned pattern'.format(id_0, id_1))
mod_int_0 = Interface(self._kwargs[rank_0]['sel'])
mod_int_0[self._kwargs[rank_0]['sel']] = 0
mod_int_1 = Interface(self._kwargs[rank_1]['sel'])
mod_int_1[self._kwargs[rank_1]['sel']] = 0
mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in'
mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out'
mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot'
mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike'
mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in'
mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out'
mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot'
mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike'
if not mod_int_0.is_compatible(0, pat.interface, int_0, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_0, int_0))
if not mod_int_1.is_compatible(0, pat.interface, int_1, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_1, int_1))
# XXX Need to check for fan-in XXX
# Store the pattern information in the routing table:
self.log_info('updating routing table with pattern')
if pat.is_connected(0, 1):
self.routing_table[id_0, id_1] = {'pattern': pat,
'int_0': int_0, 'int_1': int_1}
if pat.is_connected(1, 0):
self.routing_table[id_1, id_0] = {'pattern': pat,
'int_0': int_1, 'int_1': int_0}
self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True):
if not isinstance(pat, Pattern):
raise ValueError("pat is not a Pattern instance")
if id_0 not in self.rank_to_id.values():
raise ValueError("unrecognized module id %s" % id_0)
if id_1 not in self.rank_to_id.values():
raise ValueError("unrecognized module id %s" % id_1)
if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids):
raise ValueError("unrecognized pattern interface identifiers")
self.log_info("connecting modules {0} and {1}".format(id_0, id_1))
# Check compatibility of the interfaces exposed by the modules and the
# pattern; since the manager only contains module classes and not class
# instances, we need to create Interface instances from the selectors
# associated with the modules in order to test their compatibility:
if compat_check:
rank_0 = self.rank_to_id.inv[id_0]
rank_1 = self.rank_to_id.inv[id_1]
self.log_info("checking compatibility of modules {0} and {1} and" " assigned pattern".format(id_0, id_1))
mod_int_0 = Interface(self._kwargs[rank_0]["sel"])
mod_int_0[self._kwargs[rank_0]["sel"]] = 0
mod_int_1 = Interface(self._kwargs[rank_1]["sel"])
mod_int_1[self._kwargs[rank_1]["sel"]] = 0
mod_int_0[self._kwargs[rank_0]["sel_in"], "io"] = "in"
mod_int_0[self._kwargs[rank_0]["sel_out"], "io"] = "out"
mod_int_0[self._kwargs[rank_0]["sel_gpot"], "type"] = "gpot"
mod_int_0[self._kwargs[rank_0]["sel_spike"], "type"] = "spike"
mod_int_1[self._kwargs[rank_1]["sel_in"], "io"] = "in"
mod_int_1[self._kwargs[rank_1]["sel_out"], "io"] = "out"
mod_int_1[self._kwargs[rank_1]["sel_gpot"], "type"] = "gpot"
mod_int_1[self._kwargs[rank_1]["sel_spike"], "type"] = "spike"
if not mod_int_0.is_compatible(0, pat.interface, int_0, True):
raise ValueError("module %s interface incompatible " "with pattern interface %s" % (id_0, int_0))
if not mod_int_1.is_compatible(0, pat.interface, int_1, True):
raise ValueError("module %s interface incompatible " "with pattern interface %s" % (id_1, int_1))
# XXX Need to check for fan-in XXX
# Store the pattern information in the routing table:
self.log_info("updating routing table with pattern")
if pat.is_connected(0, 1):
self.routing_table[id_0, id_1] = {"pattern": pat, "int_0": int_0, "int_1": int_1}
if pat.is_connected(1, 0):
self.routing_table[id_1, id_0] = {"pattern": pat, "int_0": int_1, "int_1": int_0}
self.log_info("connected modules {0} and {1}".format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True):
"""
Specify connection between two module instances with a Pattern instance.
Parameters
----------
id_0, id_1 : str
Identifiers of module instances to connect.
pat : Pattern
Pattern instance.
int_0, int_1 : int
Which of the pattern's interfaces to connect to `id_0` and `id_1`,
respectively.
compat_check : bool
Check whether the interfaces of the specified modules
are compatible with the specified pattern. This option is provided
because compatibility checking can be expensive.
Notes
-----
Assumes that the constructors of the module types contain a `sel`
parameter.
"""
if not isinstance(pat, Pattern):
raise ValueError('pat is not a Pattern instance')
if id_0 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_0)
if id_1 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_1)
if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids):
raise ValueError('unrecognized pattern interface identifiers')
self.log_info('connecting modules {0} and {1}'
.format(id_0, id_1))
# Check compatibility of the interfaces exposed by the modules and the
# pattern; since the manager only contains module classes and not class
# instances, we need to create Interface instances from the selectors
# associated with the modules in order to test their compatibility:
if compat_check:
rank_0 = self.rank_to_id.inv[id_0]
rank_1 = self.rank_to_id.inv[id_1]
self.log_info('checking compatibility of modules {0} and {1} and'
' assigned pattern'.format(id_0, id_1))
mod_int_0 = Interface(self._kwargs[rank_0]['sel'])
mod_int_0[self._kwargs[rank_0]['sel']] = 0
mod_int_1 = Interface(self._kwargs[rank_1]['sel'])
mod_int_1[self._kwargs[rank_1]['sel']] = 0
mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in'
mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out'
mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot'
mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike'
mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in'
mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out'
mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot'
mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike'
if not mod_int_0.is_compatible(0, pat.interface, int_0, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_0, int_0))
if not mod_int_1.is_compatible(0, pat.interface, int_1, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_1, int_1))
# XXX Need to check for fan-in XXX
# Store the pattern information in the routing table:
self.log_info('updating routing table with pattern')
if pat.is_connected(0, 1):
self.routing_table[id_0, id_1] = {'pattern': pat,
'int_0': int_0, 'int_1': int_1}
if pat.is_connected(1, 0):
self.routing_table[id_1, id_0] = {'pattern': pat,
'int_0': int_1, 'int_1': int_0}
self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
class BaseModule(ControlledProcess):
"""
Processing module.
This class repeatedly executes a work method until it receives a
quit message via its control network port.
Parameters
----------
sel : str, unicode, or sequence
Path-like selector describing the module's interface of
exposed ports.
sel_in : str, unicode, or sequence
Selector describing all input ports in the module's interface.
sel_out : str, unicode, or sequence
Selector describing all input ports in the module's interface.
data : numpy.ndarray
Data array to associate with ports. Array length must equal the number
of ports in a module's interface.
columns : list of str
Interface port attributes.
port_data : int
Network port for transmitting data.
port_ctrl : int
Network port for controlling the module instance.
id : str
Module identifier. If no identifier is specified, a unique
identifier is automatically generated.
debug : bool
Debug flag. When True, exceptions raised during the work method
are not be suppressed.
time_sync : bool
Time synchronization flag. When True, debug messages are not emitted during
module synchronization and the time taken to receive all incoming data is
computed.
Attributes
----------
interface : Interface
Object containing information about a module's ports.
patterns : dict of Pattern
Pattern objects connecting the module instance with
other module instances. Keyed on the ID of the other module
instances.
pat_ints : dict of tuple of int
Interface of each pattern that is connected to the module instance.
Keyed on the ID of the other module instances.
pm : plsel.PortMapper
Map between a module's ports and the contents of the `data` attribute.
data : numpy.ndarray
Array of data associated with a module's ports.
Notes
-----
If the network ports specified upon instantiation are None, the module
instance ignores the network entirely.
"""
# Define properties to perform validation when connectivity status
# is set:
_net = 'none'
@property
def net(self):
"""
Network connectivity.
"""
return self._net
@net.setter
def net(self, value):
if value not in ['none', 'ctrl', 'in', 'out', 'full']:
raise ValueError('invalid network connectivity value')
self.log_info('net status changed: %s -> %s' % (self._net, value))
self._net = value
# Define properties to perform validation when the maximum number of
# execution steps set:
_max_steps = float('inf')
@property
def max_steps(self):
"""
Maximum number of steps to execute.
"""
return self._max_steps
@max_steps.setter
def max_steps(self, value):
if value <= 0:
raise ValueError('invalid maximum number of steps')
self.log_info('maximum number of steps changed: %s -> %s' % \
(self._max_steps, value))
self._max_steps = value
def __init__(self, sel, sel_in, sel_out,
data, columns=['interface', 'io', 'type'],
port_data=PORT_DATA, port_ctrl=PORT_CTRL, port_time=PORT_TIME,
id=None, debug=False, time_sync=False):
self.debug = debug
self.time_sync = time_sync
# Require several necessary attribute columns:
assert 'interface' in columns
assert 'io' in columns
assert 'type' in columns
# Generate a unique ID if none is specified:
if id is None:
id = uid()
super(BaseModule, self).__init__(port_ctrl, id)
# Reformat logger name:
LoggerMixin.__init__(self, 'mod %s' % self.id)
# Data port:
if port_data == port_ctrl:
raise ValueError('data and control ports must differ')
self.port_data = port_data
if port_time == port_ctrl or port_time == port_data:
raise ValueError('time port must differ from data and control ports')
self.port_time = port_time
# Initial network connectivity:
self.net = 'none'
# Create module interface given the specified ports:
self.interface = Interface(sel, columns)
# Set the interface ID to 0; we assume that a module only has one
# interface:
self.interface[sel, 'interface'] = 0
# Set port I/O status:
assert SelectorMethods.is_in(sel_in, sel)
assert SelectorMethods.is_in(sel_out, sel)
assert SelectorMethods.are_disjoint(sel_in, sel_out)
self.interface[sel_in, 'io'] = 'in'
self.interface[sel_out, 'io'] = 'out'
# Set up mapper between port identifiers and their associated data:
assert len(data) == len(self.interface)
self.data = data
self.pm = PortMapper(sel, self.data)
# Patterns connecting this module instance with other modules instances.
# Keyed on the IDs of those modules:
self.patterns = {}
# Each entry in pat_ints is a tuple containing the identifiers of which
# of a pattern's identifiers are connected to the current module (first
# entry) and the modules to which it is connected (second entry).
# Keyed on the IDs of those modules:
self.pat_ints = {}
# Dict for storing incoming data; each entry (corresponding to each
# module that sends input to the current module) is a deque containing
# incoming data, which in turn contains transmitted data arrays. Deques
# are used here to accommodate situations when multiple data from a
# single source arrive:
self._in_data = {}
# List for storing outgoing data; each entry is a tuple whose first
# entry is the source or destination module ID and whose second entry is
# the data to transmit:
self._out_data = []
# Dictionary containing ports of source modules that
# send output to this module. Must be initialized immediately before
# an emulation begins running. Keyed on source module ID:
self._in_port_dict = {}
# Dictionary containing ports of destination modules that
# receive input from this module. Must be initialized immediately before
# an emulation begins running. Keyed on destination module ID:
self._out_port_dict = {}
self._out_ids = []
self._in_ids = []
@property
def N_ports(self):
"""
Number of ports exposed by module's interface.
"""
return len(self.interface.ports())
@property
def all_ids(self):
"""
IDs of modules to which the current module is connected.
"""
return self.patterns.keys()
@property
def in_ids(self):
"""
IDs of modules that send data to this module.
"""
return [m for m in self.patterns.keys() \
if self.patterns[m].is_connected(self.pat_ints[m][1],
self.pat_ints[m][0])]
@property
def out_ids(self):
"""
IDs of modules that receive data from this module.
"""
return [m for m in self.patterns.keys() \
if self.patterns[m].is_connected(self.pat_ints[m][0],
self.pat_ints[m][1])]
def connect(self, m, pat, int_0, int_1, compat_check=True):
"""
Connect the current module instance to another module with a pattern instance.
Parameters
----------
m : BaseModule
Module instance to connect.
pat : Pattern
Pattern instance.
int_0, int_1 : int
Which of the pattern's interface to connect to the current module
and the specified module, respectively.
compat_check : bool
Check whether the interfaces of the current and specified modules
are compatible with the specified pattern. This option is provided
because compatibility checking can be expensive.
"""
assert isinstance(m, BaseModule)
assert isinstance(pat, Pattern)
assert int_0 in pat.interface_ids and int_1 in pat.interface_ids
self.log_info('connecting to %s' % m.id)
# Check compatibility of the interfaces exposed by the modules and the
# pattern:
if compat_check:
self.log_info('checking compatibility of modules {0} and {1} and'
' assigned pattern'.format(self.id, m.id))
assert self.interface.is_compatible(0, pat.interface, int_0, True)
assert m.interface.is_compatible(0, pat.interface, int_1, True)
# Check that no fan-in from different source modules occurs as a result
# of the new connection by getting the union of all connected input
# ports for the interfaces of all existing patterns connected to the
# current module and ensuring that the input ports from the new pattern
# don't overlap:
if self.patterns:
curr_in_ports = reduce(set.union,
[set(self.patterns[i].connected_ports(self.pat_ints[i][0]).in_ports(tuples=True)) \
for i in self.patterns.keys()])
assert not curr_in_ports.intersection(pat.connected_ports(int_0).in_ports(tuples=True))
# The pattern instances associated with the current
# module are keyed on the IDs of the modules to which they connect:
self.patterns[m.id] = pat
self.pat_ints[m.id] = (int_0, int_1)
# Update internal connectivity based upon contents of connectivity
# object. When this method is invoked, the module's internal
# connectivity is always upgraded to at least 'ctrl':
if self.net == 'none':
self.net = 'ctrl'
if pat.is_connected(int_0, int_1):
old_net = self.net
if self.net == 'ctrl':
self.net = 'out'
elif self.net == 'in':
self.net = 'full'
self.log_info('net status changed: %s -> %s' % (old_net, self.net))
if pat.is_connected(int_1, int_0):
old_net = self.net
if self.net == 'ctrl':
self.net = 'in'
elif self.net == 'out':
self.net = 'full'
self.log_info('net status changed: %s -> %s' % (old_net, self.net))
def _ctrl_stream_shutdown(self):
"""
Shut down control port handler's stream and ioloop.
"""
try:
self.stream_ctrl.flush()
self.stream_ctrl.stop_on_recv()
self.ioloop_ctrl.stop()
except IOError:
self.log_info('streams already closed')
except:
self.log_info('other error occurred')
else:
self.log_info('ctrl stream shut down')
def _ctrl_handler(self, msg):
"""
Control port handler.
"""
self.log_info('recv ctrl message: %s' % str(msg))
if msg[0] == 'quit':
self._ctrl_stream_shutdown()
# Force the module's main loop to exit:
self.running = False
ack = 'shutdown'
# One can define additional messages to be recognized by the control
# handler:
# elif msg[0] == 'conn':
# self.log_info('conn payload: '+str(msgpack.unpackb(msg[1])))
# ack = 'ack'
else:
ack = 'ack'
self.sock_ctrl.send(ack)
self.log_info('sent to manager: %s' % ack)
def _init_net(self):
"""
Initialize network connection.
"""
# Initialize control port handler:
self.log_info('initializing ctrl network connection')
super(BaseModule, self)._init_net()
# Initialize data port handler:
if self.net == 'none':
self.log_info('not initializing data network connection')
else:
# Don't allow interrupts to prevent the handler from
# completely executing each time it is called:
with IgnoreKeyboardInterrupt():
self.log_info('initializing data network connection')
# Use a nonblocking port for the data interface; set
# the linger period to prevent hanging on unsent
# messages when shutting down:
self.sock_data = self.zmq_ctx.socket(zmq.DEALER)
self.sock_data.setsockopt(zmq.IDENTITY, self.id)
self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME)
self.sock_data.connect("tcp://localhost:%i" % self.port_data)
self.log_info('data network connection initialized')
# Initialize timing port:
self.log_info('initializing time port')
self.sock_time = self.zmq_ctx.socket(zmq.DEALER)
self.sock_time.setsockopt(zmq.IDENTITY, self.id)
self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME)
self.sock_time.connect("tcp://localhost:%i" % self.port_time)
sync_dealer(self.sock_time, self.id)
self.log_info('time port initialized')
def _get_in_data(self):
"""
Get input data from incoming transmission buffer.
Populate the data array associated with a module's ports using input
data received from other modules.
"""
if self.net in ['none', 'ctrl']:
self.log_info('not retrieving from input buffer')
else:
self.log_info('retrieving from input buffer')
# Since fan-in is not permitted, the data from all source modules
# must necessarily map to different ports; we can therefore write each
# of the received data to the array associated with the module's ports
# here without worry of overwriting the data from each source module:
for in_id in self._in_ids:
# Check for exceptions so as to not fail on the first emulation
# step when there is no input data to retrieve:
try:
self.pm.set_by_inds(self._in_port_dict_ids[in_id],
self._in_data[in_id].popleft())
except:
self.log_info('no input data from [%s] retrieved' % in_id)
else:
self.log_info('input data from [%s] retrieved' % in_id)
def _put_out_data(self):
"""
Put output data in outgoing transmission buffer.
Stage data from the data array associated with a module's ports for
output to other modules.
"""
if self.net in ['none', 'ctrl']:
self.log_info('not populating output buffer')
else:
self.log_info('populating output buffer')
# Clear output buffer before populating it:
self._out_data = []
# Select data that should be sent to each destination module and append
# it to the outgoing queue:
for out_id in self._out_ids:
try:
data = self.pm.get_by_inds(self._out_port_dict_ids[out_id])
self._out_data.append((out_id, data))
except:
self.log_info('no output data to [%s] sent' % out_id)
else:
self.log_info('output data to [%s] sent' % out_id)
def _sync(self):
"""
Send output data and receive input data.
Notes
-----
Assumes that the attributes used for input and output already
exist.
Each message is a tuple containing a module ID and data; for
outbound messages, the ID is that of the destination module.
for inbound messages, the ID is that of the source module.
Data is serialized before being sent and unserialized when
received.
"""
if self.net in ['none', 'ctrl']:
self.log_info('not synchronizing with network')
else:
self.log_info('synchronizing with network')
# Send outbound data:
start = time.time()
self._put_out_data()
if self.net in ['out', 'full']:
# Send all data in outbound buffer:
send_ids = [out_id for out_id in self._out_ids]
for out_id, data in self._out_data:
self.sock_data.send(msgpack.packb((out_id, data)))
send_ids.remove(out_id)
if not self.time_sync:
self.log_info('sent to %s: %s' % (out_id, str(data)))
# Send data tuples containing None to those modules for which no
# actual data was generated to satisfy the barrier condition:
for out_id in send_ids:
self.sock_data.send(msgpack.packb((out_id, None)))
if not self.time_sync:
self.log_info('sent to %s: %s' % (out_id, None))
# All output IDs should be sent data by this point:
if not self.time_sync:
self.log_info('sent data to all output IDs')
# Receive inbound data:
if self.net in ['in', 'full']:
# Wait until inbound data is received from all source modules:
recv_ids = set(self._in_ids)
nbytes = 0
while recv_ids:
# Poll to avoid blocking:
if self.sock_data.poll(POLL_TIMEOUT):
data_packed = self.sock_data.recv()
in_id, data = msgpack.unpackb(data_packed)
if not self.time_sync:
self.log_info('recv from %s: %s' % (in_id, str(data)))
# Ignore incoming data containing None:
if data is not None:
self._in_data[in_id].append(data)
# Record number of bytes of transmitted serialized data:
nbytes += len(data_packed)
# Remove source module ID from set of IDs from which to
# expect data:
recv_ids.discard(in_id)
# Stop the synchronization if a quit message has been received:
if not self.running:
if not self.time_sync:
self.log_info('run loop stopped - stopping sync')
break
if not self.time_sync:
self.log_info('recv data from all input IDs')
self._get_in_data()
# Transmit synchronization time:
stop = time.time()
if self.time_sync:
self.log_info('sent timing data to master')
self.sock_time.send(msgpack.packb((self.id, self.steps, 'sync',
(start, stop, nbytes))))
def pre_run(self, *args, **kwargs):
"""
Code to run before main module run loop.
Code in this method will be executed after a module's process has been
launched and all connectivity objects made available, but before the
main run loop begins.
"""
self.log_info('performing pre-emulation operations')
def post_run(self, *args, **kwargs):
"""
Code to run after main module run loop.
Code in this method will be executed after a module's main loop has
terminated.
"""
self.log_info('performing post-emulation operations')
def run_step(self):
"""
Module work method.
This method should be implemented to do something interesting with new
input port data in the module's `pm` attribute and update the attribute's
output port data if necessary. It should not interact with any other
class attributes.
"""
self.log_info('running execution step')
def post_run_step(self):
"""
Code to run after each execution step.
This method can be implemented to do something immediately after each
invocation of `self.run_step()`, e.g., save generated data to a file, etc.
"""
pass
def _init_port_dicts(self):
"""
Initial dictionaries of source/destination ports in current module.
"""
# Extract identifiers of source ports in the current module's interface
# for all modules receiving output from the current module:
self._out_port_dict = {}
self._out_port_dict_ids = {}
self._out_ids = self.out_ids
for out_id in self._out_ids:
self.log_info('extracting output ports for %s' % out_id)
# Get interfaces of pattern connecting the current module to
# destination module `out_id`; `from_int` is connected to the
# current module, `to_int` is connected to the other module:
from_int, to_int = self.pat_ints[out_id]
# Get ports in interface (`from_int`) connected to the current
# module that are connected to the other module via the pattern:
self._out_port_dict[out_id] = \
self.patterns[out_id].src_idx(from_int, to_int)
self._out_port_dict_ids[out_id] = \
self.pm.ports_to_inds(self._out_port_dict[out_id])
# Extract identifiers of destination ports in the current module's
# interface for all modules sending input to the current module:
self._in_port_dict = {}
self._in_port_dict_ids = {}
self._in_ids = self.in_ids
for in_id in self._in_ids:
self.log_info('extracting input ports for %s' % in_id)
# Get interfaces of pattern connecting the current module to
# source module `out_id`; `to_int` is connected to the current
# module, `from_int` is connected to the other module:
to_int, from_int = self.pat_ints[in_id]
# Get ports in interface (`to_int`) connected to the current
# module that are connected to the other module via the pattern:
self._in_port_dict[in_id] = \
self.patterns[in_id].dest_idx(from_int, to_int)
self._in_port_dict_ids[in_id] = \
self.pm.ports_to_inds(self._in_port_dict[in_id])
def run(self):
"""
Body of process.
"""
# Don't allow keyboard interruption of process:
self.log_info('starting')
with IgnoreKeyboardInterrupt():
# Initialize environment:
self._init_net()
# Initialize _out_port_dict and _in_port_dict attributes:
self._init_port_dicts()
# Initialize Buffer for incoming data. Dict used to store the
# incoming data keyed by the source module id. Each value is a
# queue buffering the received data:
self._in_data = {k: collections.deque() for k in self.in_ids}
# Perform any pre-emulation operations:
self.pre_run()
self.running = True
self.steps = 0
if self.time_sync:
self.sock_time.send(msgpack.packb((self.id, self.steps, 'start',
time.time())))
self.log_info('sent start time to master')
while self.steps < self.max_steps:
self.log_info('execution step: %s/%s' % (self.steps, self.max_steps))
# If the debug flag is set, don't catch exceptions so that
# errors will lead to visible failures:
if self.debug:
# Run the processing step:
self.run_step()
# Do post-processing:
self.post_run_step()
# Synchronize:
self._sync()
else:
# Run the processing step:
catch_exception(self.run_step, self.log_info)
# Do post processing:
catch_exception(self.post_run_step, self.log_info)
# Synchronize:
catch_exception(self._sync, self.log_info)
# Exit run loop when a quit message has been received:
if not self.running:
self.log_info('run loop stopped')
break
self.steps += 1
if self.time_sync:
self.sock_time.send(msgpack.packb((self.id, self.steps, 'stop',
time.time())))
self.log_info('sent stop time to master')
self.log_info('maximum number of steps reached')
# Perform any post-emulation operations:
self.post_run()
# Shut down the control handler and inform the manager that the
# module has shut down:
self._ctrl_stream_shutdown()
ack = 'shutdown'
self.sock_ctrl.send(ack)
self.log_info('sent to manager: %s' % ack)
self.log_info('exiting')
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True):
"""
Specify connection between two module instances with a Pattern instance.
Parameters
----------
id_0, id_1 : str
Identifiers of module instances to connect.
pat : Pattern
Pattern instance.
int_0, int_1 : int
Which of the pattern's interfaces to connect to `id_0` and `id_1`,
respectively.
compat_check : bool
Check whether the interfaces of the specified modules
are compatible with the specified pattern. This option is provided
because compatibility checking can be expensive.
Notes
-----
Assumes that the constructors of the module types contain a `sel`
parameter.
"""
if not isinstance(pat, Pattern):
raise ValueError('pat is not a Pattern instance')
if id_0 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_0)
if id_1 not in self.rank_to_id.values():
raise ValueError('unrecognized module id %s' % id_1)
if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids):
raise ValueError('unrecognized pattern interface identifiers')
self.log_info('connecting modules {0} and {1}'.format(id_0, id_1))
# Check compatibility of the interfaces exposed by the modules and the
# pattern; since the manager only contains module classes and not class
# instances, we need to create Interface instances from the selectors
# associated with the modules in order to test their compatibility:
if compat_check:
rank_0 = self.rank_to_id.inv[id_0]
rank_1 = self.rank_to_id.inv[id_1]
self.log_info('checking compatibility of modules {0} and {1} and'
' assigned pattern'.format(id_0, id_1))
mod_int_0 = Interface(self._kwargs[rank_0]['sel'])
mod_int_0[self._kwargs[rank_0]['sel']] = 0
mod_int_1 = Interface(self._kwargs[rank_1]['sel'])
mod_int_1[self._kwargs[rank_1]['sel']] = 0
mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in'
mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out'
mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot'
mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike'
mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in'
mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out'
mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot'
mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike'
if not mod_int_0.is_compatible(0, pat.interface, int_0, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_0, int_0))
if not mod_int_1.is_compatible(0, pat.interface, int_1, True):
raise ValueError('module %s interface incompatible '
'with pattern interface %s' % (id_1, int_1))
# XXX Need to check for fan-in XXX
# Store the pattern information in the routing table:
self.log_info('updating routing table with pattern')
if pat.is_connected(0, 1):
self.routing_table[id_0, id_1] = {
'pattern': pat,
'int_0': int_0,
'int_1': int_1
}
if pat.is_connected(1, 0):
self.routing_table[id_1, id_0] = {
'pattern': pat,
'int_0': int_1,
'int_1': int_0
}
self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
class BaseModule(ControlledProcess):
"""
Processing module.
This class repeatedly executes a work method until it receives a
quit message via its control network port.
Parameters
----------
selector : str, unicode, or sequence
Path-like selector describing the module's interface of
exposed ports.
data : numpy.ndarray
Data array to associate with ports. Array length must equal the number
of ports in a module's interface.
columns : list of str
Interface port attributes.
port_data : int
Network port for transmitting data.
port_ctrl : int
Network port for controlling the module instance.
id : str
Module identifier. If no identifier is specified, a unique
identifier is automatically generated.
debug : bool
Debug flag. When True, exceptions raised during the work method
are not be suppressed.
Attributes
----------
interface : Interface
Object containing information about a module's ports.
patterns : dict of Pattern
Pattern objects connecting the module instance with
other module instances. Keyed on the ID of the other module
instances.
pat_ints : dict of tuple of int
Interface of each pattern that is connected to the module instance.
Keyed on the ID of the other module instances.
pm : plsel.PortMapper
Map between a module's ports and the contents of the `data` attribute.
data : numpy.ndarray
Array of data associated with a module's ports.
Notes
-----
If the network ports specified upon instantiation are None, the module
instance ignores the network entirely.
"""
# Define properties to perform validation when connectivity status
# is set:
_net = 'none'
@property
def net(self):
"""
Network connectivity.
"""
return self._net
@net.setter
def net(self, value):
if value not in ['none', 'ctrl', 'in', 'out', 'full']:
raise ValueError('invalid network connectivity value')
self.logger.info('net status changed: %s -> %s' % (self._net, value))
self._net = value
# Define properties to perform validation when the maximum number of
# execution steps set:
_steps = np.inf
@property
def steps(self):
"""
Maximum number of steps to execute.
"""
return self._steps
@steps.setter
def steps(self, value):
if value <= 0:
raise ValueError('invalid maximum number of steps')
self.logger.info('maximum number of steps changed: %s -> %s' %
(self._steps, value))
self._steps = value
def __init__(self,
selector,
data,
columns=['interface', 'io', 'type'],
port_data=PORT_DATA,
port_ctrl=PORT_CTRL,
id=None,
debug=False):
self.debug = debug
# Generate a unique ID if none is specified:
if id is None:
id = uid()
super(BaseModule, self).__init__(port_ctrl, id)
# Logging:
self.logger = twiggy.log.name('module %s' % self.id)
# Data port:
if port_data == port_ctrl:
raise ValueError('data and control ports must differ')
self.port_data = port_data
# Initial network connectivity:
self.net = 'none'
# Create module interface given the specified ports:
self.interface = Interface(selector, columns)
# Set the interface ID to 0; we assume that a module only has one interface:
self.interface[selector, 'interface'] = 0
# Set up mapper between port identifiers and their associated data:
assert len(data) == len(self.interface)
self.data = data
self.pm = PortMapper(self.data, selector)
# Patterns connecting this module instance with other modules instances.
# Keyed on the IDs of those modules:
self.patterns = {}
# Each entry in pat_ints is a tuple containing the identifiers of which
# of a pattern's identifiers are connected to the current module (first
# entry) and the modules to which it is connected (second entry).
# Keyed on the IDs of those modules:
self.pat_ints = {}
# Dict for storing incoming data; each entry (corresponding to each
# module that sends input to the current module) is a deque containing
# incoming data, which in turn contains transmitted data arrays. Deques
# are used here to accommodate situations when multiple data from a
# single source arrive:
self._in_data = {}
# List for storing outgoing data; each entry is a tuple whose first
# entry is the source or destination module ID and whose second entry is
# the data to transmit:
self._out_data = []
# Dictionary containing ports of source modules that
# send output to this module. Must be initialized immediately before
# an emulation begins running. Keyed on source module ID:
self._in_port_dict = {}
# Dictionary containing ports of destination modules that
# receive input from this module. Must be initialized immediately before
# an emulation begins running. Keyed on destination module ID:
self._out_port_dict = {}
self._out_ids = []
self._in_ids = []
@property
def N_ports(self):
"""
Number of ports exposed by module's interface.
"""
return len(self.interface.ports())
@property
def all_ids(self):
"""
IDs of modules to which the current module is connected.
"""
return self.patterns.keys()
@property
def in_ids(self):
"""
IDs of modules that send data to this module.
"""
return [m for m in self.patterns.keys() \
if self.patterns[m].is_connected(self.pat_ints[m][1],
self.pat_ints[m][0])]
@property
def out_ids(self):
"""
IDs of modules that receive data from this module.
"""
return [m for m in self.patterns.keys() \
if self.patterns[m].is_connected(self.pat_ints[m][0],
self.pat_ints[m][1])]
def connect(self, m, pat, int_0, int_1):
"""
Connect the current module instance to another module with a pattern instance.
Parameters
----------
m : BaseModule
Module instance to connect.
pat : Pattern
Pattern instance.
int_0, int_1 : int
Which of the pattern's interface to connect to the current module
and the specified module, respectively.
"""
assert isinstance(m, BaseModule)
assert isinstance(pat, Pattern)
assert int_0 in pat.interface_ids and int_1 in pat.interface_ids
self.logger.info('connecting to %s' % m.id)
# Check compatibility of the interfaces exposed by the modules and the
# pattern:
assert self.interface.is_compatible(0, pat.interface, int_0)
assert m.interface.is_compatible(0, pat.interface, int_1)
# Check that no fan-in from different source modules occurs as a result
# of the new connection by getting the union of all input ports for the
# interfaces of all existing patterns connected to the current module
# and ensuring that the input ports from the new pattern don't overlap:
if self.patterns:
curr_in_ports = reduce(set.union,
[set(self.patterns[i].in_ports(self.pat_ints[i][0]).to_tuples()) \
for i in self.patterns.keys()])
assert curr_in_ports.intersection(pat.in_ports(int_0).to_tuples())
# The pattern instances associated with the current
# module are keyed on the IDs of the modules to which they connect:
self.patterns[m.id] = pat
self.pat_ints[m.id] = (int_0, int_1)
# Update internal connectivity based upon contents of connectivity
# object. When this method is invoked, the module's internal
# connectivity is always upgraded to at least 'ctrl':
if self.net == 'none':
self.net = 'ctrl'
if pat.is_connected(int_0, int_1):
old_net = self.net
if self.net == 'ctrl':
self.net = 'out'
elif self.net == 'in':
self.net = 'full'
self.logger.info('net status changed: %s -> %s' %
(old_net, self.net))
if pat.is_connected(int_1, int_0):
old_net = self.net
if self.net == 'ctrl':
self.net = 'in'
elif self.net == 'out':
self.net = 'full'
self.logger.info('net status changed: %s -> %s' %
(old_net, self.net))
def _ctrl_stream_shutdown(self):
"""
Shut down control port handler's stream and ioloop.
"""
try:
self.stream_ctrl.flush()
self.stream_ctrl.stop_on_recv()
self.ioloop_ctrl.stop()
except IOError:
self.logger.info('streams already closed')
except:
self.logger.info('other error occurred')
else:
self.logger.info('ctrl stream shut down')
def _ctrl_handler(self, msg):
"""
Control port handler.
"""
self.logger.info('recv ctrl message: %s' % str(msg))
if msg[0] == 'quit':
self._ctrl_stream_shutdown()
# Force the module's main loop to exit:
self.running = False
ack = 'shutdown'
# One can define additional messages to be recognized by the control
# handler:
# elif msg[0] == 'conn':
# self.logger.info('conn payload: '+str(msgpack.unpackb(msg[1])))
# ack = 'ack'
else:
ack = 'ack'
self.sock_ctrl.send(ack)
self.logger.info('sent to manager: %s' % ack)
def _init_net(self):
"""
Initialize network connection.
"""
# Initialize control port handler:
self.logger.info('initializing ctrl network connection')
super(BaseModule, self)._init_net()
if self.net == 'none':
self.logger.info('not initializing data network connection')
else:
# Don't allow interrupts to prevent the handler from
# completely executing each time it is called:
with IgnoreKeyboardInterrupt():
self.logger.info('initializing data network connection')
# Use a nonblocking port for the data interface; set
# the linger period to prevent hanging on unsent
# messages when shutting down:
self.sock_data = self.zmq_ctx.socket(zmq.DEALER)
self.sock_data.setsockopt(zmq.IDENTITY, self.id)
self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME)
self.sock_data.connect("tcp://localhost:%i" % self.port_data)
self.logger.info('network connection initialized')
# Set up a poller for detecting incoming data:
self.data_poller = zmq.Poller()
self.data_poller.register(self.sock_data, zmq.POLLIN)
def _get_in_data(self):
"""
Get input data from incoming transmission buffer.
Populate the data array associated with a module's ports using input
data received from other modules.
"""
self.logger.info('retrieving from input buffer')
# Since fan-in is not permitted, the data from all source modules
# must necessarily map to different ports; we can therefore write each
# of the received data to the array associated with the module's ports
# here without worry of overwriting the data from each source module:
for in_id in self._in_ids:
# Check for exceptions so as to not fail on the first emulation
# step when there is no input data to retrieve:
try:
self.pm[self.
_in_port_dict[in_id]] = self._in_data[in_id].popleft()
except:
self.logger.info('no input data from [%s] retrieved' % in_id)
else:
self.logger.info('input data from [%s] retrieved' % in_id)
def _put_out_data(self):
"""
Put output data in outgoing transmission buffer.
Stage data from the data array associated with a module's ports for
output to other modules.
"""
self.logger.info('populating output buffer')
# Clear output buffer before populating it:
self._out_data = []
# Select data that should be sent to each destination module and append
# it to the outgoing queue:
for out_id in self._out_ids:
try:
self._out_data.append(
(out_id, self.pm[self._out_port_dict[out_id]]))
except:
self.logger.info('no output data to [%s] sent' % out_id)
else:
self.logger.info('output data to [%s] sent' % out_id)
def _sync(self):
"""
Send output data and receive input data.
Notes
-----
Assumes that the attributes used for input and output already
exist.
Each message is a tuple containing a module ID and data; for
outbound messages, the ID is that of the destination module.
for inbound messages, the ID is that of the source module.
Data is serialized before being sent and unserialized when
received.
"""
if self.net in ['none', 'ctrl']:
self.logger.info('not synchronizing with network')
else:
self.logger.info('synchronizing with network')
# Send outbound data:
if self.net in ['out', 'full']:
# Send all data in outbound buffer:
send_ids = [out_id for out_id in self._out_ids]
for out_id, data in self._out_data:
self.sock_data.send(msgpack.packb((out_id, data)))
send_ids.remove(out_id)
self.logger.info('sent to %s: %s' % (out_id, str(data)))
# Send data tuples containing None to those modules for which no
# actual data was generated to satisfy the barrier condition:
for out_id in send_ids:
self.sock_data.send(msgpack.packb((out_id, None)))
self.logger.info('sent to %s: %s' % (out_id, None))
# All output IDs should be sent data by this point:
self.logger.info('sent data to all output IDs')
# Receive inbound data:
if self.net in ['in', 'full']:
# Wait until inbound data is received from all source modules:
while not all((q for q in self._in_data.itervalues())):
# Use poller to avoid blocking:
if is_poll_in(self.sock_data, self.data_poller):
in_id, data = msgpack.unpackb(self.sock_data.recv())
self.logger.info('recv from %s: %s ' %
(in_id, str(data)))
# Ignore incoming data containing None:
if data is not None:
self._in_data[in_id].append(data)
# Stop the synchronization if a quit message has been received:
if not self.running:
self.logger.info('run loop stopped - stopping sync')
break
self.logger.info('recv data from all input IDs')
def pre_run(self, *args, **kwargs):
"""
Code to run before main module run loop.
Code in this method will be executed after a module's process has been
launched and all connectivity objects made available, but before the
main run loop begins.
"""
self.logger.info('performing pre-emulation operations')
def post_run(self, *args, **kwargs):
"""
Code to run after main module run loop.
Code in this method will be executed after a module's main loop has
terminated.
"""
self.logger.info('performing post-emulation operations')
def run_step(self):
"""
Module work method.
This method should be implemented to do something interesting with new
input port data in the module's `pm` attribute and update the attribute's
output port data if necessary. It should not interact with any other
class attributes.
"""
self.logger.info('running execution step')
def _init_port_dicts(self):
"""
Initial dictionaries of source/destination ports in current module.
"""
# Extract identifiers of source ports in the current module's interface
# for all modules receiving output from the current module:
self._out_port_dict = {}
self._out_ids = self.out_ids
for out_id in self._out_ids:
self.logger.info('extracting output ports for %s' % out_id)
# Get interfaces of pattern connecting the current module to
# destination module `out_id`; `from_int` is connected to the
# current module, `to_int` is connected to the other module:
from_int, to_int = self.pat_ints[out_id]
# Get ports in interface (`from_int`) connected to the current
# module that are connected to the other module via the pattern:
self._out_port_dict[out_id] = \
self.patterns[out_id].src_idx(from_int, to_int)
# Extract identifiers of destination ports in the current module's
# interface for all modules sending input to the current module:
self._in_port_dict = {}
self._in_ids = self.in_ids
for in_id in self._in_ids:
self.logger.info('extracting input ports for %s' % in_id)
# Get interfaces of pattern connecting the current module to
# source module `out_id`; `to_int` is connected to the current
# module, `from_int` is connected to the other module:
to_int, from_int = self.pat_ints[in_id]
# Get ports in interface (`to_int`) connected to the current
# module that are connected to the other module via the pattern:
self._in_port_dict[in_id] = \
self.patterns[in_id].dest_idx(from_int, to_int)
def run(self):
"""
Body of process.
"""
# Don't allow keyboard interruption of process:
self.logger.info('starting')
with IgnoreKeyboardInterrupt():
# Initialize environment:
self._init_net()
# Initialize _out_port_dict and _in_port_dict attributes:
self._init_port_dicts()
# Initialize Buffer for incoming data. Dict used to store the
# incoming data keyed by the source module id. Each value is a
# queue buffering the received data:
self._in_data = {k: collections.deque() for k in self.in_ids}
# Perform any pre-emulation operations:
self.pre_run()
self.running = True
curr_steps = 0
while curr_steps < self._steps:
self.logger.info('execution step: %s' % curr_steps)
# If the debug flag is set, don't catch exceptions so that
# errors will lead to visible failures:
if self.debug:
# Get input data:
self._get_in_data()
# Run the processing step:
self.run_step()
# Prepare the generated data for output:
self._put_out_data()
# Synchronize:
self._sync()
else:
# Get input data:
catch_exception(self._get_in_data, self.logger.info)
# Run the processing step:
catch_exception(self.run_step, self.logger.info)
# Prepare the generated data for output:
catch_exception(self._put_out_data, self.logger.info)
# Synchronize:
catch_exception(self._sync, self.logger.info)
# Exit run loop when a quit message has been received:
if not self.running:
self.logger.info('run loop stopped')
break
curr_steps += 1
# Perform any post-emulation operations:
self.post_run()
# Shut down the control handler and inform the manager that the
# module has shut down:
self._ctrl_stream_shutdown()
ack = 'shutdown'
self.sock_ctrl.send(ack)
self.logger.info('sent to manager: %s' % ack)
self.logger.info('exiting')
