def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True): if not isinstance(pat, Pattern): raise ValueError('pat is not a Pattern instance') if id_0 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_0) if id_1 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_1) if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids): raise ValueError('unrecognized pattern interface identifiers') self.log_info('connecting modules {0} and {1}'.format(id_0, id_1)) # Check compatibility of the interfaces exposed by the modules and the # pattern; since the manager only contains module classes and not class # instances, we need to create Interface instances from the selectors # associated with the modules in order to test their compatibility: if compat_check: rank_0 = self.rank_to_id.inv[id_0] rank_1 = self.rank_to_id.inv[id_1] self.log_info('checking compatibility of modules {0} and {1} and' ' assigned pattern'.format(id_0, id_1)) mod_int_0 = Interface(self._kwargs[rank_0]['sel']) mod_int_0[self._kwargs[rank_0]['sel']] = 0 mod_int_1 = Interface(self._kwargs[rank_1]['sel']) mod_int_1[self._kwargs[rank_1]['sel']] = 0 mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in' mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out' mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot' mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike' mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in' mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out' mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot' mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike' if not mod_int_0.is_compatible(0, pat.interface, int_0, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_0, int_0)) if not mod_int_1.is_compatible(0, pat.interface, int_1, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_1, int_1)) # XXX Need to check for fan-in XXX # Store the pattern information in the routing table: self.log_info('updating routing table with pattern') if pat.is_connected(0, 1): self.routing_table[id_0, id_1] = { 'pattern': pat, 'int_0': int_0, 'int_1': int_1 } if pat.is_connected(1, 0): self.routing_table[id_1, id_0] = { 'pattern': pat, 'int_0': int_1, 'int_1': int_0 } self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True): if not isinstance(pat, Pattern): raise ValueError('pat is not a Pattern instance') if id_0 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_0) if id_1 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_1) if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids): raise ValueError('unrecognized pattern interface identifiers') self.log_info('connecting modules {0} and {1}' .format(id_0, id_1)) # Check compatibility of the interfaces exposed by the modules and the # pattern; since the manager only contains module classes and not class # instances, we need to create Interface instances from the selectors # associated with the modules in order to test their compatibility: if compat_check: rank_0 = self.rank_to_id.inv[id_0] rank_1 = self.rank_to_id.inv[id_1] self.log_info('checking compatibility of modules {0} and {1} and' ' assigned pattern'.format(id_0, id_1)) mod_int_0 = Interface(self._kwargs[rank_0]['sel']) mod_int_0[self._kwargs[rank_0]['sel']] = 0 mod_int_1 = Interface(self._kwargs[rank_1]['sel']) mod_int_1[self._kwargs[rank_1]['sel']] = 0 mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in' mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out' mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot' mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike' mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in' mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out' mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot' mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike' if not mod_int_0.is_compatible(0, pat.interface, int_0, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_0, int_0)) if not mod_int_1.is_compatible(0, pat.interface, int_1, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_1, int_1)) # XXX Need to check for fan-in XXX # Store the pattern information in the routing table: self.log_info('updating routing table with pattern') if pat.is_connected(0, 1): self.routing_table[id_0, id_1] = {'pattern': pat, 'int_0': int_0, 'int_1': int_1} if pat.is_connected(1, 0): self.routing_table[id_1, id_0] = {'pattern': pat, 'int_0': int_1, 'int_1': int_0} self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True): if not isinstance(pat, Pattern): raise ValueError("pat is not a Pattern instance") if id_0 not in self.rank_to_id.values(): raise ValueError("unrecognized module id %s" % id_0) if id_1 not in self.rank_to_id.values(): raise ValueError("unrecognized module id %s" % id_1) if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids): raise ValueError("unrecognized pattern interface identifiers") self.log_info("connecting modules {0} and {1}".format(id_0, id_1)) # Check compatibility of the interfaces exposed by the modules and the # pattern; since the manager only contains module classes and not class # instances, we need to create Interface instances from the selectors # associated with the modules in order to test their compatibility: if compat_check: rank_0 = self.rank_to_id.inv[id_0] rank_1 = self.rank_to_id.inv[id_1] self.log_info("checking compatibility of modules {0} and {1} and" " assigned pattern".format(id_0, id_1)) mod_int_0 = Interface(self._kwargs[rank_0]["sel"]) mod_int_0[self._kwargs[rank_0]["sel"]] = 0 mod_int_1 = Interface(self._kwargs[rank_1]["sel"]) mod_int_1[self._kwargs[rank_1]["sel"]] = 0 mod_int_0[self._kwargs[rank_0]["sel_in"], "io"] = "in" mod_int_0[self._kwargs[rank_0]["sel_out"], "io"] = "out" mod_int_0[self._kwargs[rank_0]["sel_gpot"], "type"] = "gpot" mod_int_0[self._kwargs[rank_0]["sel_spike"], "type"] = "spike" mod_int_1[self._kwargs[rank_1]["sel_in"], "io"] = "in" mod_int_1[self._kwargs[rank_1]["sel_out"], "io"] = "out" mod_int_1[self._kwargs[rank_1]["sel_gpot"], "type"] = "gpot" mod_int_1[self._kwargs[rank_1]["sel_spike"], "type"] = "spike" if not mod_int_0.is_compatible(0, pat.interface, int_0, True): raise ValueError("module %s interface incompatible " "with pattern interface %s" % (id_0, int_0)) if not mod_int_1.is_compatible(0, pat.interface, int_1, True): raise ValueError("module %s interface incompatible " "with pattern interface %s" % (id_1, int_1)) # XXX Need to check for fan-in XXX # Store the pattern information in the routing table: self.log_info("updating routing table with pattern") if pat.is_connected(0, 1): self.routing_table[id_0, id_1] = {"pattern": pat, "int_0": int_0, "int_1": int_1} if pat.is_connected(1, 0): self.routing_table[id_1, id_0] = {"pattern": pat, "int_0": int_1, "int_1": int_0} self.log_info("connected modules {0} and {1}".format(id_0, id_1))
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True): """ Specify connection between two module instances with a Pattern instance. Parameters ---------- id_0, id_1 : str Identifiers of module instances to connect. pat : Pattern Pattern instance. int_0, int_1 : int Which of the pattern's interfaces to connect to `id_0` and `id_1`, respectively. compat_check : bool Check whether the interfaces of the specified modules are compatible with the specified pattern. This option is provided because compatibility checking can be expensive. Notes ----- Assumes that the constructors of the module types contain a `sel` parameter. """ if not isinstance(pat, Pattern): raise ValueError('pat is not a Pattern instance') if id_0 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_0) if id_1 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_1) if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids): raise ValueError('unrecognized pattern interface identifiers') self.log_info('connecting modules {0} and {1}' .format(id_0, id_1)) # Check compatibility of the interfaces exposed by the modules and the # pattern; since the manager only contains module classes and not class # instances, we need to create Interface instances from the selectors # associated with the modules in order to test their compatibility: if compat_check: rank_0 = self.rank_to_id.inv[id_0] rank_1 = self.rank_to_id.inv[id_1] self.log_info('checking compatibility of modules {0} and {1} and' ' assigned pattern'.format(id_0, id_1)) mod_int_0 = Interface(self._kwargs[rank_0]['sel']) mod_int_0[self._kwargs[rank_0]['sel']] = 0 mod_int_1 = Interface(self._kwargs[rank_1]['sel']) mod_int_1[self._kwargs[rank_1]['sel']] = 0 mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in' mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out' mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot' mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike' mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in' mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out' mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot' mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike' if not mod_int_0.is_compatible(0, pat.interface, int_0, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_0, int_0)) if not mod_int_1.is_compatible(0, pat.interface, int_1, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_1, int_1)) # XXX Need to check for fan-in XXX # Store the pattern information in the routing table: self.log_info('updating routing table with pattern') if pat.is_connected(0, 1): self.routing_table[id_0, id_1] = {'pattern': pat, 'int_0': int_0, 'int_1': int_1} if pat.is_connected(1, 0): self.routing_table[id_1, id_0] = {'pattern': pat, 'int_0': int_1, 'int_1': int_0} self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
class BaseModule(ControlledProcess): """ Processing module. This class repeatedly executes a work method until it receives a quit message via its control network port. Parameters ---------- sel : str, unicode, or sequence Path-like selector describing the module's interface of exposed ports. sel_in : str, unicode, or sequence Selector describing all input ports in the module's interface. sel_out : str, unicode, or sequence Selector describing all input ports in the module's interface. data : numpy.ndarray Data array to associate with ports. Array length must equal the number of ports in a module's interface. columns : list of str Interface port attributes. port_data : int Network port for transmitting data. port_ctrl : int Network port for controlling the module instance. id : str Module identifier. If no identifier is specified, a unique identifier is automatically generated. debug : bool Debug flag. When True, exceptions raised during the work method are not be suppressed. time_sync : bool Time synchronization flag. When True, debug messages are not emitted during module synchronization and the time taken to receive all incoming data is computed. Attributes ---------- interface : Interface Object containing information about a module's ports. patterns : dict of Pattern Pattern objects connecting the module instance with other module instances. Keyed on the ID of the other module instances. pat_ints : dict of tuple of int Interface of each pattern that is connected to the module instance. Keyed on the ID of the other module instances. pm : plsel.PortMapper Map between a module's ports and the contents of the `data` attribute. data : numpy.ndarray Array of data associated with a module's ports. Notes ----- If the network ports specified upon instantiation are None, the module instance ignores the network entirely. """ # Define properties to perform validation when connectivity status # is set: _net = 'none' @property def net(self): """ Network connectivity. """ return self._net @net.setter def net(self, value): if value not in ['none', 'ctrl', 'in', 'out', 'full']: raise ValueError('invalid network connectivity value') self.log_info('net status changed: %s -> %s' % (self._net, value)) self._net = value # Define properties to perform validation when the maximum number of # execution steps set: _max_steps = float('inf') @property def max_steps(self): """ Maximum number of steps to execute. """ return self._max_steps @max_steps.setter def max_steps(self, value): if value <= 0: raise ValueError('invalid maximum number of steps') self.log_info('maximum number of steps changed: %s -> %s' % \ (self._max_steps, value)) self._max_steps = value def __init__(self, sel, sel_in, sel_out, data, columns=['interface', 'io', 'type'], port_data=PORT_DATA, port_ctrl=PORT_CTRL, port_time=PORT_TIME, id=None, debug=False, time_sync=False): self.debug = debug self.time_sync = time_sync # Require several necessary attribute columns: assert 'interface' in columns assert 'io' in columns assert 'type' in columns # Generate a unique ID if none is specified: if id is None: id = uid() super(BaseModule, self).__init__(port_ctrl, id) # Reformat logger name: LoggerMixin.__init__(self, 'mod %s' % self.id) # Data port: if port_data == port_ctrl: raise ValueError('data and control ports must differ') self.port_data = port_data if port_time == port_ctrl or port_time == port_data: raise ValueError('time port must differ from data and control ports') self.port_time = port_time # Initial network connectivity: self.net = 'none' # Create module interface given the specified ports: self.interface = Interface(sel, columns) # Set the interface ID to 0; we assume that a module only has one # interface: self.interface[sel, 'interface'] = 0 # Set port I/O status: assert SelectorMethods.is_in(sel_in, sel) assert SelectorMethods.is_in(sel_out, sel) assert SelectorMethods.are_disjoint(sel_in, sel_out) self.interface[sel_in, 'io'] = 'in' self.interface[sel_out, 'io'] = 'out' # Set up mapper between port identifiers and their associated data: assert len(data) == len(self.interface) self.data = data self.pm = PortMapper(sel, self.data) # Patterns connecting this module instance with other modules instances. # Keyed on the IDs of those modules: self.patterns = {} # Each entry in pat_ints is a tuple containing the identifiers of which # of a pattern's identifiers are connected to the current module (first # entry) and the modules to which it is connected (second entry). # Keyed on the IDs of those modules: self.pat_ints = {} # Dict for storing incoming data; each entry (corresponding to each # module that sends input to the current module) is a deque containing # incoming data, which in turn contains transmitted data arrays. Deques # are used here to accommodate situations when multiple data from a # single source arrive: self._in_data = {} # List for storing outgoing data; each entry is a tuple whose first # entry is the source or destination module ID and whose second entry is # the data to transmit: self._out_data = [] # Dictionary containing ports of source modules that # send output to this module. Must be initialized immediately before # an emulation begins running. Keyed on source module ID: self._in_port_dict = {} # Dictionary containing ports of destination modules that # receive input from this module. Must be initialized immediately before # an emulation begins running. Keyed on destination module ID: self._out_port_dict = {} self._out_ids = [] self._in_ids = [] @property def N_ports(self): """ Number of ports exposed by module's interface. """ return len(self.interface.ports()) @property def all_ids(self): """ IDs of modules to which the current module is connected. """ return self.patterns.keys() @property def in_ids(self): """ IDs of modules that send data to this module. """ return [m for m in self.patterns.keys() \ if self.patterns[m].is_connected(self.pat_ints[m][1], self.pat_ints[m][0])] @property def out_ids(self): """ IDs of modules that receive data from this module. """ return [m for m in self.patterns.keys() \ if self.patterns[m].is_connected(self.pat_ints[m][0], self.pat_ints[m][1])] def connect(self, m, pat, int_0, int_1, compat_check=True): """ Connect the current module instance to another module with a pattern instance. Parameters ---------- m : BaseModule Module instance to connect. pat : Pattern Pattern instance. int_0, int_1 : int Which of the pattern's interface to connect to the current module and the specified module, respectively. compat_check : bool Check whether the interfaces of the current and specified modules are compatible with the specified pattern. This option is provided because compatibility checking can be expensive. """ assert isinstance(m, BaseModule) assert isinstance(pat, Pattern) assert int_0 in pat.interface_ids and int_1 in pat.interface_ids self.log_info('connecting to %s' % m.id) # Check compatibility of the interfaces exposed by the modules and the # pattern: if compat_check: self.log_info('checking compatibility of modules {0} and {1} and' ' assigned pattern'.format(self.id, m.id)) assert self.interface.is_compatible(0, pat.interface, int_0, True) assert m.interface.is_compatible(0, pat.interface, int_1, True) # Check that no fan-in from different source modules occurs as a result # of the new connection by getting the union of all connected input # ports for the interfaces of all existing patterns connected to the # current module and ensuring that the input ports from the new pattern # don't overlap: if self.patterns: curr_in_ports = reduce(set.union, [set(self.patterns[i].connected_ports(self.pat_ints[i][0]).in_ports(tuples=True)) \ for i in self.patterns.keys()]) assert not curr_in_ports.intersection(pat.connected_ports(int_0).in_ports(tuples=True)) # The pattern instances associated with the current # module are keyed on the IDs of the modules to which they connect: self.patterns[m.id] = pat self.pat_ints[m.id] = (int_0, int_1) # Update internal connectivity based upon contents of connectivity # object. When this method is invoked, the module's internal # connectivity is always upgraded to at least 'ctrl': if self.net == 'none': self.net = 'ctrl' if pat.is_connected(int_0, int_1): old_net = self.net if self.net == 'ctrl': self.net = 'out' elif self.net == 'in': self.net = 'full' self.log_info('net status changed: %s -> %s' % (old_net, self.net)) if pat.is_connected(int_1, int_0): old_net = self.net if self.net == 'ctrl': self.net = 'in' elif self.net == 'out': self.net = 'full' self.log_info('net status changed: %s -> %s' % (old_net, self.net)) def _ctrl_stream_shutdown(self): """ Shut down control port handler's stream and ioloop. """ try: self.stream_ctrl.flush() self.stream_ctrl.stop_on_recv() self.ioloop_ctrl.stop() except IOError: self.log_info('streams already closed') except: self.log_info('other error occurred') else: self.log_info('ctrl stream shut down') def _ctrl_handler(self, msg): """ Control port handler. """ self.log_info('recv ctrl message: %s' % str(msg)) if msg[0] == 'quit': self._ctrl_stream_shutdown() # Force the module's main loop to exit: self.running = False ack = 'shutdown' # One can define additional messages to be recognized by the control # handler: # elif msg[0] == 'conn': # self.log_info('conn payload: '+str(msgpack.unpackb(msg[1]))) # ack = 'ack' else: ack = 'ack' self.sock_ctrl.send(ack) self.log_info('sent to manager: %s' % ack) def _init_net(self): """ Initialize network connection. """ # Initialize control port handler: self.log_info('initializing ctrl network connection') super(BaseModule, self)._init_net() # Initialize data port handler: if self.net == 'none': self.log_info('not initializing data network connection') else: # Don't allow interrupts to prevent the handler from # completely executing each time it is called: with IgnoreKeyboardInterrupt(): self.log_info('initializing data network connection') # Use a nonblocking port for the data interface; set # the linger period to prevent hanging on unsent # messages when shutting down: self.sock_data = self.zmq_ctx.socket(zmq.DEALER) self.sock_data.setsockopt(zmq.IDENTITY, self.id) self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME) self.sock_data.connect("tcp://localhost:%i" % self.port_data) self.log_info('data network connection initialized') # Initialize timing port: self.log_info('initializing time port') self.sock_time = self.zmq_ctx.socket(zmq.DEALER) self.sock_time.setsockopt(zmq.IDENTITY, self.id) self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME) self.sock_time.connect("tcp://localhost:%i" % self.port_time) sync_dealer(self.sock_time, self.id) self.log_info('time port initialized') def _get_in_data(self): """ Get input data from incoming transmission buffer. Populate the data array associated with a module's ports using input data received from other modules. """ if self.net in ['none', 'ctrl']: self.log_info('not retrieving from input buffer') else: self.log_info('retrieving from input buffer') # Since fan-in is not permitted, the data from all source modules # must necessarily map to different ports; we can therefore write each # of the received data to the array associated with the module's ports # here without worry of overwriting the data from each source module: for in_id in self._in_ids: # Check for exceptions so as to not fail on the first emulation # step when there is no input data to retrieve: try: self.pm.set_by_inds(self._in_port_dict_ids[in_id], self._in_data[in_id].popleft()) except: self.log_info('no input data from [%s] retrieved' % in_id) else: self.log_info('input data from [%s] retrieved' % in_id) def _put_out_data(self): """ Put output data in outgoing transmission buffer. Stage data from the data array associated with a module's ports for output to other modules. """ if self.net in ['none', 'ctrl']: self.log_info('not populating output buffer') else: self.log_info('populating output buffer') # Clear output buffer before populating it: self._out_data = [] # Select data that should be sent to each destination module and append # it to the outgoing queue: for out_id in self._out_ids: try: data = self.pm.get_by_inds(self._out_port_dict_ids[out_id]) self._out_data.append((out_id, data)) except: self.log_info('no output data to [%s] sent' % out_id) else: self.log_info('output data to [%s] sent' % out_id) def _sync(self): """ Send output data and receive input data. Notes ----- Assumes that the attributes used for input and output already exist. Each message is a tuple containing a module ID and data; for outbound messages, the ID is that of the destination module. for inbound messages, the ID is that of the source module. Data is serialized before being sent and unserialized when received. """ if self.net in ['none', 'ctrl']: self.log_info('not synchronizing with network') else: self.log_info('synchronizing with network') # Send outbound data: start = time.time() self._put_out_data() if self.net in ['out', 'full']: # Send all data in outbound buffer: send_ids = [out_id for out_id in self._out_ids] for out_id, data in self._out_data: self.sock_data.send(msgpack.packb((out_id, data))) send_ids.remove(out_id) if not self.time_sync: self.log_info('sent to %s: %s' % (out_id, str(data))) # Send data tuples containing None to those modules for which no # actual data was generated to satisfy the barrier condition: for out_id in send_ids: self.sock_data.send(msgpack.packb((out_id, None))) if not self.time_sync: self.log_info('sent to %s: %s' % (out_id, None)) # All output IDs should be sent data by this point: if not self.time_sync: self.log_info('sent data to all output IDs') # Receive inbound data: if self.net in ['in', 'full']: # Wait until inbound data is received from all source modules: recv_ids = set(self._in_ids) nbytes = 0 while recv_ids: # Poll to avoid blocking: if self.sock_data.poll(POLL_TIMEOUT): data_packed = self.sock_data.recv() in_id, data = msgpack.unpackb(data_packed) if not self.time_sync: self.log_info('recv from %s: %s' % (in_id, str(data))) # Ignore incoming data containing None: if data is not None: self._in_data[in_id].append(data) # Record number of bytes of transmitted serialized data: nbytes += len(data_packed) # Remove source module ID from set of IDs from which to # expect data: recv_ids.discard(in_id) # Stop the synchronization if a quit message has been received: if not self.running: if not self.time_sync: self.log_info('run loop stopped - stopping sync') break if not self.time_sync: self.log_info('recv data from all input IDs') self._get_in_data() # Transmit synchronization time: stop = time.time() if self.time_sync: self.log_info('sent timing data to master') self.sock_time.send(msgpack.packb((self.id, self.steps, 'sync', (start, stop, nbytes)))) def pre_run(self, *args, **kwargs): """ Code to run before main module run loop. Code in this method will be executed after a module's process has been launched and all connectivity objects made available, but before the main run loop begins. """ self.log_info('performing pre-emulation operations') def post_run(self, *args, **kwargs): """ Code to run after main module run loop. Code in this method will be executed after a module's main loop has terminated. """ self.log_info('performing post-emulation operations') def run_step(self): """ Module work method. This method should be implemented to do something interesting with new input port data in the module's `pm` attribute and update the attribute's output port data if necessary. It should not interact with any other class attributes. """ self.log_info('running execution step') def post_run_step(self): """ Code to run after each execution step. This method can be implemented to do something immediately after each invocation of `self.run_step()`, e.g., save generated data to a file, etc. """ pass def _init_port_dicts(self): """ Initial dictionaries of source/destination ports in current module. """ # Extract identifiers of source ports in the current module's interface # for all modules receiving output from the current module: self._out_port_dict = {} self._out_port_dict_ids = {} self._out_ids = self.out_ids for out_id in self._out_ids: self.log_info('extracting output ports for %s' % out_id) # Get interfaces of pattern connecting the current module to # destination module `out_id`; `from_int` is connected to the # current module, `to_int` is connected to the other module: from_int, to_int = self.pat_ints[out_id] # Get ports in interface (`from_int`) connected to the current # module that are connected to the other module via the pattern: self._out_port_dict[out_id] = \ self.patterns[out_id].src_idx(from_int, to_int) self._out_port_dict_ids[out_id] = \ self.pm.ports_to_inds(self._out_port_dict[out_id]) # Extract identifiers of destination ports in the current module's # interface for all modules sending input to the current module: self._in_port_dict = {} self._in_port_dict_ids = {} self._in_ids = self.in_ids for in_id in self._in_ids: self.log_info('extracting input ports for %s' % in_id) # Get interfaces of pattern connecting the current module to # source module `out_id`; `to_int` is connected to the current # module, `from_int` is connected to the other module: to_int, from_int = self.pat_ints[in_id] # Get ports in interface (`to_int`) connected to the current # module that are connected to the other module via the pattern: self._in_port_dict[in_id] = \ self.patterns[in_id].dest_idx(from_int, to_int) self._in_port_dict_ids[in_id] = \ self.pm.ports_to_inds(self._in_port_dict[in_id]) def run(self): """ Body of process. """ # Don't allow keyboard interruption of process: self.log_info('starting') with IgnoreKeyboardInterrupt(): # Initialize environment: self._init_net() # Initialize _out_port_dict and _in_port_dict attributes: self._init_port_dicts() # Initialize Buffer for incoming data. Dict used to store the # incoming data keyed by the source module id. Each value is a # queue buffering the received data: self._in_data = {k: collections.deque() for k in self.in_ids} # Perform any pre-emulation operations: self.pre_run() self.running = True self.steps = 0 if self.time_sync: self.sock_time.send(msgpack.packb((self.id, self.steps, 'start', time.time()))) self.log_info('sent start time to master') while self.steps < self.max_steps: self.log_info('execution step: %s/%s' % (self.steps, self.max_steps)) # If the debug flag is set, don't catch exceptions so that # errors will lead to visible failures: if self.debug: # Run the processing step: self.run_step() # Do post-processing: self.post_run_step() # Synchronize: self._sync() else: # Run the processing step: catch_exception(self.run_step, self.log_info) # Do post processing: catch_exception(self.post_run_step, self.log_info) # Synchronize: catch_exception(self._sync, self.log_info) # Exit run loop when a quit message has been received: if not self.running: self.log_info('run loop stopped') break self.steps += 1 if self.time_sync: self.sock_time.send(msgpack.packb((self.id, self.steps, 'stop', time.time()))) self.log_info('sent stop time to master') self.log_info('maximum number of steps reached') # Perform any post-emulation operations: self.post_run() # Shut down the control handler and inform the manager that the # module has shut down: self._ctrl_stream_shutdown() ack = 'shutdown' self.sock_ctrl.send(ack) self.log_info('sent to manager: %s' % ack) self.log_info('exiting')
def connect(self, id_0, id_1, pat, int_0=0, int_1=1, compat_check=True): """ Specify connection between two module instances with a Pattern instance. Parameters ---------- id_0, id_1 : str Identifiers of module instances to connect. pat : Pattern Pattern instance. int_0, int_1 : int Which of the pattern's interfaces to connect to `id_0` and `id_1`, respectively. compat_check : bool Check whether the interfaces of the specified modules are compatible with the specified pattern. This option is provided because compatibility checking can be expensive. Notes ----- Assumes that the constructors of the module types contain a `sel` parameter. """ if not isinstance(pat, Pattern): raise ValueError('pat is not a Pattern instance') if id_0 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_0) if id_1 not in self.rank_to_id.values(): raise ValueError('unrecognized module id %s' % id_1) if not (int_0 in pat.interface_ids and int_1 in pat.interface_ids): raise ValueError('unrecognized pattern interface identifiers') self.log_info('connecting modules {0} and {1}'.format(id_0, id_1)) # Check compatibility of the interfaces exposed by the modules and the # pattern; since the manager only contains module classes and not class # instances, we need to create Interface instances from the selectors # associated with the modules in order to test their compatibility: if compat_check: rank_0 = self.rank_to_id.inv[id_0] rank_1 = self.rank_to_id.inv[id_1] self.log_info('checking compatibility of modules {0} and {1} and' ' assigned pattern'.format(id_0, id_1)) mod_int_0 = Interface(self._kwargs[rank_0]['sel']) mod_int_0[self._kwargs[rank_0]['sel']] = 0 mod_int_1 = Interface(self._kwargs[rank_1]['sel']) mod_int_1[self._kwargs[rank_1]['sel']] = 0 mod_int_0[self._kwargs[rank_0]['sel_in'], 'io'] = 'in' mod_int_0[self._kwargs[rank_0]['sel_out'], 'io'] = 'out' mod_int_0[self._kwargs[rank_0]['sel_gpot'], 'type'] = 'gpot' mod_int_0[self._kwargs[rank_0]['sel_spike'], 'type'] = 'spike' mod_int_1[self._kwargs[rank_1]['sel_in'], 'io'] = 'in' mod_int_1[self._kwargs[rank_1]['sel_out'], 'io'] = 'out' mod_int_1[self._kwargs[rank_1]['sel_gpot'], 'type'] = 'gpot' mod_int_1[self._kwargs[rank_1]['sel_spike'], 'type'] = 'spike' if not mod_int_0.is_compatible(0, pat.interface, int_0, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_0, int_0)) if not mod_int_1.is_compatible(0, pat.interface, int_1, True): raise ValueError('module %s interface incompatible ' 'with pattern interface %s' % (id_1, int_1)) # XXX Need to check for fan-in XXX # Store the pattern information in the routing table: self.log_info('updating routing table with pattern') if pat.is_connected(0, 1): self.routing_table[id_0, id_1] = { 'pattern': pat, 'int_0': int_0, 'int_1': int_1 } if pat.is_connected(1, 0): self.routing_table[id_1, id_0] = { 'pattern': pat, 'int_0': int_1, 'int_1': int_0 } self.log_info('connected modules {0} and {1}'.format(id_0, id_1))
class BaseModule(ControlledProcess): """ Processing module. This class repeatedly executes a work method until it receives a quit message via its control network port. Parameters ---------- selector : str, unicode, or sequence Path-like selector describing the module's interface of exposed ports. data : numpy.ndarray Data array to associate with ports. Array length must equal the number of ports in a module's interface. columns : list of str Interface port attributes. port_data : int Network port for transmitting data. port_ctrl : int Network port for controlling the module instance. id : str Module identifier. If no identifier is specified, a unique identifier is automatically generated. debug : bool Debug flag. When True, exceptions raised during the work method are not be suppressed. Attributes ---------- interface : Interface Object containing information about a module's ports. patterns : dict of Pattern Pattern objects connecting the module instance with other module instances. Keyed on the ID of the other module instances. pat_ints : dict of tuple of int Interface of each pattern that is connected to the module instance. Keyed on the ID of the other module instances. pm : plsel.PortMapper Map between a module's ports and the contents of the `data` attribute. data : numpy.ndarray Array of data associated with a module's ports. Notes ----- If the network ports specified upon instantiation are None, the module instance ignores the network entirely. """ # Define properties to perform validation when connectivity status # is set: _net = 'none' @property def net(self): """ Network connectivity. """ return self._net @net.setter def net(self, value): if value not in ['none', 'ctrl', 'in', 'out', 'full']: raise ValueError('invalid network connectivity value') self.logger.info('net status changed: %s -> %s' % (self._net, value)) self._net = value # Define properties to perform validation when the maximum number of # execution steps set: _steps = np.inf @property def steps(self): """ Maximum number of steps to execute. """ return self._steps @steps.setter def steps(self, value): if value <= 0: raise ValueError('invalid maximum number of steps') self.logger.info('maximum number of steps changed: %s -> %s' % (self._steps, value)) self._steps = value def __init__(self, selector, data, columns=['interface', 'io', 'type'], port_data=PORT_DATA, port_ctrl=PORT_CTRL, id=None, debug=False): self.debug = debug # Generate a unique ID if none is specified: if id is None: id = uid() super(BaseModule, self).__init__(port_ctrl, id) # Logging: self.logger = twiggy.log.name('module %s' % self.id) # Data port: if port_data == port_ctrl: raise ValueError('data and control ports must differ') self.port_data = port_data # Initial network connectivity: self.net = 'none' # Create module interface given the specified ports: self.interface = Interface(selector, columns) # Set the interface ID to 0; we assume that a module only has one interface: self.interface[selector, 'interface'] = 0 # Set up mapper between port identifiers and their associated data: assert len(data) == len(self.interface) self.data = data self.pm = PortMapper(self.data, selector) # Patterns connecting this module instance with other modules instances. # Keyed on the IDs of those modules: self.patterns = {} # Each entry in pat_ints is a tuple containing the identifiers of which # of a pattern's identifiers are connected to the current module (first # entry) and the modules to which it is connected (second entry). # Keyed on the IDs of those modules: self.pat_ints = {} # Dict for storing incoming data; each entry (corresponding to each # module that sends input to the current module) is a deque containing # incoming data, which in turn contains transmitted data arrays. Deques # are used here to accommodate situations when multiple data from a # single source arrive: self._in_data = {} # List for storing outgoing data; each entry is a tuple whose first # entry is the source or destination module ID and whose second entry is # the data to transmit: self._out_data = [] # Dictionary containing ports of source modules that # send output to this module. Must be initialized immediately before # an emulation begins running. Keyed on source module ID: self._in_port_dict = {} # Dictionary containing ports of destination modules that # receive input from this module. Must be initialized immediately before # an emulation begins running. Keyed on destination module ID: self._out_port_dict = {} self._out_ids = [] self._in_ids = [] @property def N_ports(self): """ Number of ports exposed by module's interface. """ return len(self.interface.ports()) @property def all_ids(self): """ IDs of modules to which the current module is connected. """ return self.patterns.keys() @property def in_ids(self): """ IDs of modules that send data to this module. """ return [m for m in self.patterns.keys() \ if self.patterns[m].is_connected(self.pat_ints[m][1], self.pat_ints[m][0])] @property def out_ids(self): """ IDs of modules that receive data from this module. """ return [m for m in self.patterns.keys() \ if self.patterns[m].is_connected(self.pat_ints[m][0], self.pat_ints[m][1])] def connect(self, m, pat, int_0, int_1): """ Connect the current module instance to another module with a pattern instance. Parameters ---------- m : BaseModule Module instance to connect. pat : Pattern Pattern instance. int_0, int_1 : int Which of the pattern's interface to connect to the current module and the specified module, respectively. """ assert isinstance(m, BaseModule) assert isinstance(pat, Pattern) assert int_0 in pat.interface_ids and int_1 in pat.interface_ids self.logger.info('connecting to %s' % m.id) # Check compatibility of the interfaces exposed by the modules and the # pattern: assert self.interface.is_compatible(0, pat.interface, int_0) assert m.interface.is_compatible(0, pat.interface, int_1) # Check that no fan-in from different source modules occurs as a result # of the new connection by getting the union of all input ports for the # interfaces of all existing patterns connected to the current module # and ensuring that the input ports from the new pattern don't overlap: if self.patterns: curr_in_ports = reduce(set.union, [set(self.patterns[i].in_ports(self.pat_ints[i][0]).to_tuples()) \ for i in self.patterns.keys()]) assert curr_in_ports.intersection(pat.in_ports(int_0).to_tuples()) # The pattern instances associated with the current # module are keyed on the IDs of the modules to which they connect: self.patterns[m.id] = pat self.pat_ints[m.id] = (int_0, int_1) # Update internal connectivity based upon contents of connectivity # object. When this method is invoked, the module's internal # connectivity is always upgraded to at least 'ctrl': if self.net == 'none': self.net = 'ctrl' if pat.is_connected(int_0, int_1): old_net = self.net if self.net == 'ctrl': self.net = 'out' elif self.net == 'in': self.net = 'full' self.logger.info('net status changed: %s -> %s' % (old_net, self.net)) if pat.is_connected(int_1, int_0): old_net = self.net if self.net == 'ctrl': self.net = 'in' elif self.net == 'out': self.net = 'full' self.logger.info('net status changed: %s -> %s' % (old_net, self.net)) def _ctrl_stream_shutdown(self): """ Shut down control port handler's stream and ioloop. """ try: self.stream_ctrl.flush() self.stream_ctrl.stop_on_recv() self.ioloop_ctrl.stop() except IOError: self.logger.info('streams already closed') except: self.logger.info('other error occurred') else: self.logger.info('ctrl stream shut down') def _ctrl_handler(self, msg): """ Control port handler. """ self.logger.info('recv ctrl message: %s' % str(msg)) if msg[0] == 'quit': self._ctrl_stream_shutdown() # Force the module's main loop to exit: self.running = False ack = 'shutdown' # One can define additional messages to be recognized by the control # handler: # elif msg[0] == 'conn': # self.logger.info('conn payload: '+str(msgpack.unpackb(msg[1]))) # ack = 'ack' else: ack = 'ack' self.sock_ctrl.send(ack) self.logger.info('sent to manager: %s' % ack) def _init_net(self): """ Initialize network connection. """ # Initialize control port handler: self.logger.info('initializing ctrl network connection') super(BaseModule, self)._init_net() if self.net == 'none': self.logger.info('not initializing data network connection') else: # Don't allow interrupts to prevent the handler from # completely executing each time it is called: with IgnoreKeyboardInterrupt(): self.logger.info('initializing data network connection') # Use a nonblocking port for the data interface; set # the linger period to prevent hanging on unsent # messages when shutting down: self.sock_data = self.zmq_ctx.socket(zmq.DEALER) self.sock_data.setsockopt(zmq.IDENTITY, self.id) self.sock_data.setsockopt(zmq.LINGER, LINGER_TIME) self.sock_data.connect("tcp://localhost:%i" % self.port_data) self.logger.info('network connection initialized') # Set up a poller for detecting incoming data: self.data_poller = zmq.Poller() self.data_poller.register(self.sock_data, zmq.POLLIN) def _get_in_data(self): """ Get input data from incoming transmission buffer. Populate the data array associated with a module's ports using input data received from other modules. """ self.logger.info('retrieving from input buffer') # Since fan-in is not permitted, the data from all source modules # must necessarily map to different ports; we can therefore write each # of the received data to the array associated with the module's ports # here without worry of overwriting the data from each source module: for in_id in self._in_ids: # Check for exceptions so as to not fail on the first emulation # step when there is no input data to retrieve: try: self.pm[self. _in_port_dict[in_id]] = self._in_data[in_id].popleft() except: self.logger.info('no input data from [%s] retrieved' % in_id) else: self.logger.info('input data from [%s] retrieved' % in_id) def _put_out_data(self): """ Put output data in outgoing transmission buffer. Stage data from the data array associated with a module's ports for output to other modules. """ self.logger.info('populating output buffer') # Clear output buffer before populating it: self._out_data = [] # Select data that should be sent to each destination module and append # it to the outgoing queue: for out_id in self._out_ids: try: self._out_data.append( (out_id, self.pm[self._out_port_dict[out_id]])) except: self.logger.info('no output data to [%s] sent' % out_id) else: self.logger.info('output data to [%s] sent' % out_id) def _sync(self): """ Send output data and receive input data. Notes ----- Assumes that the attributes used for input and output already exist. Each message is a tuple containing a module ID and data; for outbound messages, the ID is that of the destination module. for inbound messages, the ID is that of the source module. Data is serialized before being sent and unserialized when received. """ if self.net in ['none', 'ctrl']: self.logger.info('not synchronizing with network') else: self.logger.info('synchronizing with network') # Send outbound data: if self.net in ['out', 'full']: # Send all data in outbound buffer: send_ids = [out_id for out_id in self._out_ids] for out_id, data in self._out_data: self.sock_data.send(msgpack.packb((out_id, data))) send_ids.remove(out_id) self.logger.info('sent to %s: %s' % (out_id, str(data))) # Send data tuples containing None to those modules for which no # actual data was generated to satisfy the barrier condition: for out_id in send_ids: self.sock_data.send(msgpack.packb((out_id, None))) self.logger.info('sent to %s: %s' % (out_id, None)) # All output IDs should be sent data by this point: self.logger.info('sent data to all output IDs') # Receive inbound data: if self.net in ['in', 'full']: # Wait until inbound data is received from all source modules: while not all((q for q in self._in_data.itervalues())): # Use poller to avoid blocking: if is_poll_in(self.sock_data, self.data_poller): in_id, data = msgpack.unpackb(self.sock_data.recv()) self.logger.info('recv from %s: %s ' % (in_id, str(data))) # Ignore incoming data containing None: if data is not None: self._in_data[in_id].append(data) # Stop the synchronization if a quit message has been received: if not self.running: self.logger.info('run loop stopped - stopping sync') break self.logger.info('recv data from all input IDs') def pre_run(self, *args, **kwargs): """ Code to run before main module run loop. Code in this method will be executed after a module's process has been launched and all connectivity objects made available, but before the main run loop begins. """ self.logger.info('performing pre-emulation operations') def post_run(self, *args, **kwargs): """ Code to run after main module run loop. Code in this method will be executed after a module's main loop has terminated. """ self.logger.info('performing post-emulation operations') def run_step(self): """ Module work method. This method should be implemented to do something interesting with new input port data in the module's `pm` attribute and update the attribute's output port data if necessary. It should not interact with any other class attributes. """ self.logger.info('running execution step') def _init_port_dicts(self): """ Initial dictionaries of source/destination ports in current module. """ # Extract identifiers of source ports in the current module's interface # for all modules receiving output from the current module: self._out_port_dict = {} self._out_ids = self.out_ids for out_id in self._out_ids: self.logger.info('extracting output ports for %s' % out_id) # Get interfaces of pattern connecting the current module to # destination module `out_id`; `from_int` is connected to the # current module, `to_int` is connected to the other module: from_int, to_int = self.pat_ints[out_id] # Get ports in interface (`from_int`) connected to the current # module that are connected to the other module via the pattern: self._out_port_dict[out_id] = \ self.patterns[out_id].src_idx(from_int, to_int) # Extract identifiers of destination ports in the current module's # interface for all modules sending input to the current module: self._in_port_dict = {} self._in_ids = self.in_ids for in_id in self._in_ids: self.logger.info('extracting input ports for %s' % in_id) # Get interfaces of pattern connecting the current module to # source module `out_id`; `to_int` is connected to the current # module, `from_int` is connected to the other module: to_int, from_int = self.pat_ints[in_id] # Get ports in interface (`to_int`) connected to the current # module that are connected to the other module via the pattern: self._in_port_dict[in_id] = \ self.patterns[in_id].dest_idx(from_int, to_int) def run(self): """ Body of process. """ # Don't allow keyboard interruption of process: self.logger.info('starting') with IgnoreKeyboardInterrupt(): # Initialize environment: self._init_net() # Initialize _out_port_dict and _in_port_dict attributes: self._init_port_dicts() # Initialize Buffer for incoming data. Dict used to store the # incoming data keyed by the source module id. Each value is a # queue buffering the received data: self._in_data = {k: collections.deque() for k in self.in_ids} # Perform any pre-emulation operations: self.pre_run() self.running = True curr_steps = 0 while curr_steps < self._steps: self.logger.info('execution step: %s' % curr_steps) # If the debug flag is set, don't catch exceptions so that # errors will lead to visible failures: if self.debug: # Get input data: self._get_in_data() # Run the processing step: self.run_step() # Prepare the generated data for output: self._put_out_data() # Synchronize: self._sync() else: # Get input data: catch_exception(self._get_in_data, self.logger.info) # Run the processing step: catch_exception(self.run_step, self.logger.info) # Prepare the generated data for output: catch_exception(self._put_out_data, self.logger.info) # Synchronize: catch_exception(self._sync, self.logger.info) # Exit run loop when a quit message has been received: if not self.running: self.logger.info('run loop stopped') break curr_steps += 1 # Perform any post-emulation operations: self.post_run() # Shut down the control handler and inform the manager that the # module has shut down: self._ctrl_stream_shutdown() ack = 'shutdown' self.sock_ctrl.send(ack) self.logger.info('sent to manager: %s' % ack) self.logger.info('exiting')