def test_gpot_ports(self):
i = Interface('/foo[0:6]')
i['/foo[0]'] = [0, 'in', 'spike']
i['/foo[1:3]'] = [0, 'out', 'spike']
i['/foo[3]'] = [0, 'in', 'gpot']
i['/foo[4:6]'] = [0, 'out', 'gpot']
j = Interface('/foo[3:6]')
j['/foo[3]'] = [0, 'in', 'gpot']
j['/foo[4:6]'] = [0, 'out', 'gpot']
# Test returning result as Interface:
assert_frame_equal(i.gpot_ports(0).data, j.data)
# Test returning result as list of tuples:
self.assertItemsEqual(i.gpot_ports(0, True), j.data.index.tolist())
def test_gpot_ports(self):
i = Interface('/foo[0:6]')
i['/foo[0]'] = [0, 'in', 'spike']
i['/foo[1:3]'] = [0, 'out', 'spike']
i['/foo[3]'] = [0, 'in', 'gpot']
i['/foo[4:6]'] = [0, 'out', 'gpot']
j = Interface('/foo[3:6]')
j['/foo[3]'] = [0, 'in', 'gpot']
j['/foo[4:6]'] = [0, 'out', 'gpot']
# Test returning result as Interface:
assert_frame_equal(i.gpot_ports(0).data, j.data)
# Test returning result as list of tuples:
self.assertItemsEqual(i.gpot_ports(0, True),
j.data.index.tolist())
def test_gpot_ports(self):
i = Interface('/foo[0:6]')
i['/foo[0]'] = [0, 'in', 'spike']
i['/foo[1:3]'] = [0, 'out', 'spike']
i['/foo[3]'] = [0, 'in', 'gpot']
i['/foo[4:6]'] = [0, 'out', 'gpot']
j = Interface('/foo[3:6]')
j['/foo[3]'] = [0, 'in', 'gpot']
j['/foo[4:6]'] = [0, 'out', 'gpot']
assert_frame_equal(i.gpot_ports(0).data, j.data)
assert_index_equal(i.gpot_ports(0).index, j.index)
class Module(mpi.Worker):
"""
Processing module.
This class repeatedly executes a work method until it receives a
quit message via its control network port.
Parameters
----------
sel : str, unicode, or sequence
Path-like selector describing the module's interface of
exposed ports.
sel_in, sel_out, sel_gpot, sel_spike : str, unicode, or sequence
Selectors respectively describing all input, output, graded potential,
and spiking ports in the module's interface.
data_gpot, data_spike : numpy.ndarray
Data arrays associated with the graded potential and spiking ports in
the . Array length must equal the number
of ports in a module's interface.
columns : list of str
Interface port attributes.
Network port for controlling the module instance.
ctrl_tag, gpot_tag, spike_tag : int
MPI tags that respectively identify messages containing control data,
graded potential port values, and spiking port values transmitted to
worker nodes.
id : str
Module identifier. If no identifier is specified, a unique
identifier is automatically generated.
device : int
GPU device to use. May be set to None if the module does not perform
GPU processing.
routing_table : neurokernel.routing_table.RoutingTable
Routing table describing data connections between modules. If no routing
table is specified, the module will be executed in isolation.
rank_to_id : bidict.bidict
Mapping between MPI ranks and module object IDs.
debug : bool
Debug flag. When True, exceptions raised during the work method
are not be suppressed.
time_sync : bool
Time synchronization flag. When True, debug messages are not emitted
during module synchronization and the time taken to receive all incoming
data is computed.
Attributes
----------
interface : Interface
Object containing information about a module's ports.
pm : dict
`pm['gpot']` and `pm['spike']` are instances of neurokernel.pm_gpu.PortMapper that
map a module's ports to the contents of the values in `data`.
data : dict
`data['gpot']` and `data['spike']` are arrays of data associated with
a module's graded potential and spiking ports.
"""
def __init__(self, sel, sel_in, sel_out,
sel_gpot, sel_spike, data_gpot, data_spike,
columns=['interface', 'io', 'type'],
ctrl_tag=CTRL_TAG, gpot_tag=GPOT_TAG, spike_tag=SPIKE_TAG,
id=None, device=None,
routing_table=None, rank_to_id=None,
debug=False, time_sync=False, print_timing=False):
super(Module, self).__init__(ctrl_tag)
self.debug = debug
self.time_sync = time_sync
self.device = device
self.print_timing = print_timing
self._gpot_tag = gpot_tag
self._spike_tag = spike_tag
# Require several necessary attribute columns:
if 'interface' not in columns:
raise ValueError('interface column required')
if 'io' not in columns:
raise ValueError('io column required')
if 'type' not in columns:
raise ValueError('type column required')
# Initialize GPU here so as to be able to initialize a port mapper
# containing GPU memory:
self._init_gpu()
# This is needed to ensure that MPI_Finalize is called before PyCUDA
# attempts to clean up; see
# https://groups.google.com/forum/#!topic/mpi4py/by0Rd5q0Ayw
atexit.register(MPI.Finalize)
# Manually register the file close method associated with MPIOutput
# so that it is called by atexit before MPI.Finalize() (if the file is
# closed after MPI.Finalize() is called, an error will occur):
for k, v in iteritems(twiggy.emitters):
if isinstance(v._output, MPIOutput):
atexit.register(v._output.close)
# Ensure that the input and output port selectors respectively
# select mutually exclusive subsets of the set of all ports exposed by
# the module:
if not SelectorMethods.is_in(sel_in, sel):
raise ValueError('input port selector not in selector of all ports')
if not SelectorMethods.is_in(sel_out, sel):
raise ValueError('output port selector not in selector of all ports')
if not SelectorMethods.are_disjoint(sel_in, sel_out):
raise ValueError('input and output port selectors not disjoint')
#assert(SelectorMethods.is_in(sel_in, sel))
#assert(SelectorMethods.is_in(sel_out, sel))
#assert(SelectorMethods.are_disjoint(sel_in, sel_out))
# Ensure that the graded potential and spiking port selectors
# respectively select mutually exclusive subsets of the set of all ports
# exposed by the module:
#assert(SelectorMethods.is_in(sel_gpot, sel))
#assert(SelectorMethods.is_in(sel_spike, sel))
#assert(SelectorMethods.are_disjoint(sel_gpot, sel_spike))
# Save routing table and mapping between MPI ranks and module IDs:
self.routing_table = routing_table
self.rank_to_id = rank_to_id
# Generate a unique ID if none is specified:
if id is None:
self.id = uid()
else:
# If a unique ID was specified and the routing table is not empty
# (i.e., there are connections between multiple modules), the id
# must be a node in the routing table:
if routing_table is not None and len(routing_table.ids) and \
not routing_table.has_node(id):
raise ValueError('routing table must contain specified '
'module ID: {}'.format(id))
self.id = id
# Reformat logger name:
LoggerMixin.__init__(self, 'mod %s' % self.id)
if self.print_timing:
start = time.time()
# Create module interface given the specified ports:
self.interface = Interface(sel, columns)
# Set the interface ID to 0; we assume that a module only has one interface:
self.interface[sel, 'interface'] = 0
# Set the port attributes:
if len(sel_in):
self.interface[sel_in, 'io'] = 'in'
if len(sel_out):
self.interface[sel_out, 'io'] = 'out'
if len(sel_gpot):
self.interface[sel_gpot, 'type'] = 'gpot'
if len(sel_spike):
self.interface[sel_spike, 'type'] = 'spike'
if self.print_timing:
self.log_info('Elapsed time for setting up interface: {:.3f} seconds'.format(time.time()-start))
start = time.time()
# Find the graded potential and spiking ports:
self.gpot_ports = self.interface.gpot_ports()
self.spike_ports = self.interface.spike_ports()
if len(self.gpot_ports):
self.in_gpot_ports = self.gpot_ports.in_ports(tuples=True)
self.out_gpot_ports = self.gpot_ports.out_ports(tuples=True)
else:
self.in_gpot_ports = []
self.out_gpot_ports = []
if len(self.spike_ports):
self.in_spike_ports = self.spike_ports.in_ports(tuples=True)
self.out_spike_ports = self.spike_ports.out_ports(tuples=True)
else:
self.in_spike_ports = []
self.out_spike_ports = []
if self.print_timing:
self.log_info('Elapsed time for extracting ports: {:.3f} seconds'.format(time.time()-start))
start = time.time()
# Set up mapper between port identifiers and their associated data:
if len(data_gpot) != len(self.gpot_ports):
raise ValueError('incompatible gpot port data array length')
if len(data_spike) != len(self.spike_ports):
raise ValueError('incompatible spike port data array length')
self.data = {}
self.data['gpot'] = gpuarray.to_gpu(data_gpot)
self.data['spike'] = gpuarray.to_gpu(data_spike)
self.pm = {}
self.pm['gpot'] = GPUPortMapper(sel_gpot, self.data['gpot'], make_copy=False)
self.pm['spike'] = GPUPortMapper(sel_spike, self.data['spike'], make_copy=False)
if self.print_timing:
cuda.Context.synchronize()
self.log_info('Elapsed time for creating array and PortMapper {} seconds'.format(time.time()-start))
def _init_gpu(self):
"""
Initialize GPU device.
Notes
-----
Must be called from within the `run()` method, not from within
`__init__()`.
"""
if self.device == None:
self.log_info('no GPU specified - not initializing ')
else:
# Import pycuda.driver here so as to facilitate the
# subclassing of Module to create pure Python LPUs that don't use GPUs:
import pycuda.driver as drv
drv.init()
N_gpu = drv.Device.count()
if not self.device < N_gpu:
new_device = randint(0,N_gpu - 1)
self.log_warning("GPU device device %d not in GPU devices %s" % (self.device, str(range(0,N_gpu))))
self.log_warning("Setting device = %d" % new_device)
self.device = new_device
try:
self.gpu_ctx = drv.Device(self.device).make_context()
except Exception as e:
self.log_info('_init_gpu exception: ' + e.message)
else:
atexit.register(self.gpu_ctx.pop)
self.log_info('GPU %s initialized' % self.device)
def _init_port_dicts(self):
"""
Initial dictionaries of source/destination ports in current module.
"""
if self.print_timing:
start = time.time()
# Extract identifiers of source ports in the current module's interface
# for all modules receiving output from the current module:
self._out_port_dict_ids = {}
self._out_port_dict_ids['gpot'] = {}
self._out_port_dict_ids['spike'] = {}
self._out_ids = self.routing_table.dest_ids(self.id)
self._out_ranks = [self.rank_to_id.inv[i] for i in self._out_ids]
for out_id in self._out_ids:
self.log_info('extracting output ports for %s' % out_id)
# Get interfaces of pattern connecting the current module to
# destination module `out_id`; `int_0` is connected to the
# current module, `int_1` is connected to the other module:
pat = self.routing_table[self.id, out_id]['pattern']
int_0 = self.routing_table[self.id, out_id]['int_0']
int_1 = self.routing_table[self.id, out_id]['int_1']
# Get ports in interface (`int_0`) connected to the current
# module that are connected to the other module via the pattern:
self._out_port_dict_ids['gpot'][out_id] = \
gpuarray.to_gpu(self.pm['gpot'].ports_to_inds(pat.src_idx(int_0, int_1, 'gpot', 'gpot')))
self._out_port_dict_ids['spike'][out_id] = \
gpuarray.to_gpu(self.pm['spike'].ports_to_inds(pat.src_idx(int_0, int_1, 'spike', 'spike')))
if self.print_timing:
cuda.Context.synchronize()
self.log_info('Elapsed time for extracting output ports: {:.3f} seconds'.format(time.time()-start))
start = time.time()
# Extract identifiers of destination ports in the current module's
# interface for all modules sending input to the current module:
self._in_port_dict_ids = {}
self._in_port_dict_ids['gpot'] = {}
self._in_port_dict_ids['spike'] = {}
# Extract indices corresponding to the entries in the transmitted
# buffers that must be copied into the input port map data arrays; these
# are needed to support fan-out:
self._in_port_dict_buf_ids = {}
self._in_port_dict_buf_ids['gpot'] = {}
self._in_port_dict_buf_ids['spike'] = {}
# Lengths of input buffers:
self._in_buf_len = {}
self._in_buf_len['gpot'] = {}
self._in_buf_len['spike'] = {}
self._in_ids = self.routing_table.src_ids(self.id)
self._in_ranks = [self.rank_to_id.inv[i] for i in self._in_ids]
for in_id in self._in_ids:
self.log_info('extracting input ports for %s' % in_id)
# Get interfaces of pattern connecting the current module to
# source module `in_id`; `int_1` is connected to the current
# module, `int_0` is connected to the other module:
pat = self.routing_table[in_id, self.id]['pattern']
int_0 = self.routing_table[in_id, self.id]['int_0']
int_1 = self.routing_table[in_id, self.id]['int_1']
# Get ports in interface (`int_1`) connected to the current
# module that are connected to the other module via the pattern:
self._in_port_dict_ids['gpot'][in_id] = \
gpuarray.to_gpu(self.pm['gpot'].ports_to_inds(pat.dest_idx(int_0, int_1, 'gpot', 'gpot')))
self._in_port_dict_ids['spike'][in_id] = \
gpuarray.to_gpu(self.pm['spike'].ports_to_inds(pat.dest_idx(int_0, int_1, 'spike', 'spike')))
# Get the integer indices associated with the connected source ports
# in the pattern interface connected to the source module `in_d`;
# these are needed to copy received buffer contents into the current
# module's port map data array:
src_idx_dup = pat.src_idx(int_0, int_1, 'gpot', 'gpot', duplicates=True)
src_idx = OrderedDict.fromkeys(src_idx_dup).keys()
self._in_port_dict_buf_ids['gpot'][in_id] = \
np.array(renumber_in_order(BasePortMapper(pat.gpot_ports(int_0).to_tuples()).
ports_to_inds(src_idx_dup)))
src_idx_dup_s = pat.src_idx(int_0, int_1, 'spike', 'spike', duplicates=True)
src_idx_s = OrderedDict.fromkeys(src_idx_dup_s).keys()
self._in_port_dict_buf_ids['spike'][in_id] = \
np.array(renumber_in_order(BasePortMapper(pat.spike_ports(int_0).to_tuples()).
ports_to_inds(src_idx_dup_s)))
# The size of the input buffer to the current module must be the
# same length as the output buffer of module `in_id`:
self._in_buf_len['gpot'][in_id] = len(src_idx)
self._in_buf_len['spike'][in_id] = len(src_idx_s)
if self.print_timing:
self.log_info('Elapsed time for extracting input ports: {:.3f} seconds'.format(time.time()-start))
def _init_comm_bufs(self):
"""
Buffers for sending/receiving data from other modules.
Notes
-----
Must be executed after `_init_port_dicts()`.
"""
# Buffers (and their interfaces and MPI types) for receiving data
# transmitted from source modules:
self._in_buf = {}
self._in_buf['gpot'] = {}
self._in_buf['spike'] = {}
self._in_buf_int = {}
self._in_buf_int['gpot'] = {}
self._in_buf_int['spike'] = {}
self._in_buf_mtype = {}
self._in_buf_mtype['gpot'] = {}
self._in_buf_mtype['spike'] = {}
for in_id in self._in_ids:
n_gpot = self._in_buf_len['gpot'][in_id]
if n_gpot:
self._in_buf['gpot'][in_id] = \
gpuarray.empty(n_gpot, self.pm['gpot'].dtype)
self._in_buf_int['gpot'][in_id] = \
bufint(self._in_buf['gpot'][in_id])
self._in_buf_mtype['gpot'][in_id] = \
dtype_to_mpi(self._in_buf['gpot'][in_id].dtype)
else:
self._in_buf['gpot'][in_id] = None
n_spike = self._in_buf_len['spike'][in_id]
if n_spike:
self._in_buf['spike'][in_id] = \
gpuarray.empty(n_spike, self.pm['spike'].dtype)
self._in_buf_int['spike'][in_id] = \
bufint(self._in_buf['spike'][in_id])
self._in_buf_mtype['spike'][in_id] = \
dtype_to_mpi(self._in_buf['spike'][in_id].dtype)
else:
self._in_buf['spike'][in_id] = None
# Buffers (and their interfaces and MPI types) for transmitting data to
# destination modules:
self._out_buf = {}
self._out_buf['gpot'] = {}
self._out_buf['spike'] = {}
self._out_buf_int = {}
self._out_buf_int['gpot'] = {}
self._out_buf_int['spike'] = {}
self._out_buf_mtype = {}
self._out_buf_mtype['gpot'] = {}
self._out_buf_mtype['spike'] = {}
for out_id in self._out_ids:
n_gpot = len(self._out_port_dict_ids['gpot'][out_id])
if n_gpot:
self._out_buf['gpot'][out_id] = \
gpuarray.empty(n_gpot, self.pm['gpot'].dtype)
self._out_buf_int['gpot'][out_id] = \
bufint(self._out_buf['gpot'][out_id])
self._out_buf_mtype['gpot'][out_id] = \
dtype_to_mpi(self._out_buf['gpot'][out_id].dtype)
else:
self._out_buf['gpot'][out_id] = None
n_spike = len(self._out_port_dict_ids['spike'][out_id])
if n_spike:
self._out_buf['spike'][out_id] = \
gpuarray.empty(n_spike, self.pm['spike'].dtype)
self._out_buf_int['spike'][out_id] = \
bufint(self._out_buf['spike'][out_id])
self._out_buf_mtype['spike'][out_id] = \
dtype_to_mpi(self._out_buf['spike'][out_id].dtype)
else:
self._out_buf['spike'][out_id] = None
def _sync(self):
"""
Send output data and receive input data.
"""
if self.time_sync:
start = time.time()
requests = []
# For each destination module, extract elements from the current
# module's port data array, copy them to a contiguous array, and
# transmit the latter:
for dest_id, dest_rank in zip(self._out_ids, self._out_ranks):
# Copy data into destination buffer:
if self._out_buf['gpot'][dest_id] is not None:
set_by_inds(self._out_buf['gpot'][dest_id],
self._out_port_dict_ids['gpot'][dest_id],
self.data['gpot'], 'src')
if not self.time_sync:
self.log_info('gpot data sent to %s: %s' % \
(dest_id, str(self._out_buf['gpot'][dest_id])))
r = MPI.COMM_WORLD.Isend([self._out_buf_int['gpot'][dest_id],
self._out_buf_mtype['gpot'][dest_id]],
dest_rank, GPOT_TAG)
requests.append(r)
if self._out_buf['spike'][dest_id] is not None:
set_by_inds(self._out_buf['spike'][dest_id],
self._out_port_dict_ids['spike'][dest_id],
self.data['spike'], 'src')
if not self.time_sync:
self.log_info('spike data sent to %s: %s' % \
(dest_id, str(self._out_buf['spike'][dest_id])))
r = MPI.COMM_WORLD.Isend([self._out_buf_int['spike'][dest_id],
self._out_buf_mtype['spike'][dest_id]],
dest_rank, SPIKE_TAG)
requests.append(r)
if not self.time_sync:
self.log_info('sending to %s' % dest_id)
if not self.time_sync:
self.log_info('sent all data from %s' % self.id)
# For each source module, receive elements and copy them into the
# current module's port data array:
for src_id, src_rank in zip(self._in_ids, self._in_ranks):
if self._in_buf['gpot'][src_id] is not None:
r = MPI.COMM_WORLD.Irecv([self._in_buf_int['gpot'][src_id],
self._in_buf_mtype['gpot'][src_id]],
source=src_rank, tag=GPOT_TAG)
requests.append(r)
if self._in_buf['spike'][src_id] is not None:
r = MPI.COMM_WORLD.Irecv([self._in_buf_int['spike'][src_id],
self._in_buf_mtype['spike'][src_id]],
source=src_rank, tag=SPIKE_TAG)
requests.append(r)
if not self.time_sync:
self.log_info('receiving from %s' % src_id)
if requests:
req = MPI.Request()
req.Waitall(requests)
if not self.time_sync:
self.log_info('all data were received by %s' % self.id)
# Copy received elements into the current module's data array:
for src_id in self._in_ids:
if self._in_buf['gpot'][src_id] is not None:
if not self.time_sync:
self.log_info('gpot data received from %s: %s' % \
(src_id, str(self._in_buf['gpot'][src_id])))
set_by_inds_from_inds(self.data['gpot'],
self._in_port_dict_ids['gpot'][src_id],
self._in_buf['gpot'][src_id],
self._in_port_dict_buf_ids['gpot'][src_id])
if self._in_buf['spike'][src_id] is not None:
if not self.time_sync:
self.log_info('spike data received from %s: %s' % \
(src_id, str(self._in_buf['spike'][src_id])))
set_by_inds_from_inds(self.data['spike'],
self._in_port_dict_ids['spike'][src_id],
self._in_buf['spike'][src_id],
self._in_port_dict_buf_ids['spike'][src_id])
# Save timing data:
if self.time_sync:
stop = time.time()
n_gpot = 0
n_spike = 0
for src_id in self._in_ids:
n_gpot += len(self._in_buf['gpot'][src_id])
n_spike += len(self._in_buf['spike'][src_id])
self.log_info('sent timing data to master')
self.intercomm.isend(['sync_time',
(self.rank, self.steps, start, stop,
n_gpot*self.pm['gpot'].dtype.itemsize+\
n_spike*self.pm['spike'].dtype.itemsize)],
dest=0, tag=self._ctrl_tag)
else:
self.log_info('saved all data received by %s' % self.id)
def pre_run(self):
"""
Code to run before main loop.
This method is invoked by the `run()` method before the main loop is
started.
"""
# MPI Request object for resolving asynchronous transfers:
#self.req = MPI.Request()
self.log_info('running code before body of worker %s' % self.rank)
# Initialize _out_port_dict and _in_port_dict attributes:
self._init_port_dicts()
# Initialize transmission buffers:
self._init_comm_bufs()
# Start timing the main loop:
if self.time_sync:
self.intercomm.isend(['start_time', (self.rank, time.time())],
dest=0, tag=self._ctrl_tag)
self.log_info('sent start time to manager')
def post_run(self):
"""
Code to run after main loop.
This method is invoked by the `run()` method after the main loop is
started.
"""
self.log_info('running code after body of worker %s' % self.rank)
# Stop timing the main loop before shutting down the emulation:
if self.time_sync:
self.intercomm.isend(['stop_time', (self.rank, time.time())],
dest=0, tag=self._ctrl_tag)
self.log_info('sent stop time to manager')
# Send acknowledgment message:
self.intercomm.isend(['done', self.rank], 0, self._ctrl_tag)
self.log_info('done message sent to manager')
def run_step(self):
"""
Module work method.
This method should be implemented to do something interesting with new
input port data in the module's `pm` attribute and update the attribute's
output port data if necessary. It should not interact with any other
class attributes.
"""
self.log_info('running execution step')
def run(self):
"""
Body of process.
"""
# Don't allow keyboard interruption of process:
with IgnoreKeyboardInterrupt():
# Activate execution loop:
super(Module, self).run()
def do_work(self):
"""
Work method.
This method is repeatedly executed by the Worker instance after the
instance receives a 'start' control message and until it receives a 'stop'
control message.
"""
# If the debug flag is set, don't catch exceptions so that
# errors will lead to visible failures:
if self.debug:
# Run the processing step:
self.run_step()
# Synchronize:
self._sync()
else:
# Run the processing step:
catch_exception(self.run_step, self.log_info)
# Synchronize:
catch_exception(self._sync, self.log_info)