def __init__(self, key_prefix, count=0, data=None, offset=0):
EIEIOWithPayloadDataMessage.__init__(
self, EIEIODataHeader(EIEIOType.KEY_PAYLOAD_16_BIT,
prefix=key_prefix, is_time=True,
count=count),
data, offset)
def __init__(self, key_prefix, count=0, data=None, offset=0):
EIEIOWithPayloadDataMessage.__init__(
self,
EIEIODataHeader(EIEIOType.KEY_PAYLOAD_32_BIT,
prefix=key_prefix,
prefix_type=EIEIOPrefix.UPPER_HALF_WORD,
count=count), data, offset)
def read_eieio_data_message(data, offset):
""" Reads the content of an EIEIO data message and returns an object\
identifying the data which was contained in the packet
:param data: data received from the network
:type data: bytestring
:param offset: offset at which the parsing operation should start
:type offset: int
:return: an object which inherits from EIEIODataMessage which contains\
parsed data received from the network
:rtype:\
:py:class:`spinnman.messages.eieio.data_messages.eieio_data_message.EIEIODataMessage`
"""
eieio_header = EIEIODataHeader.from_bytestring(data, offset)
offset += eieio_header.size
eieio_type = eieio_header.eieio_type
prefix = eieio_header.prefix
payload_base = eieio_header.payload_base
prefix_type = eieio_header.prefix_type
is_time = eieio_header.is_time
if eieio_type == EIEIOType.KEY_16_BIT:
return _read_16_bit_message(prefix, payload_base, prefix_type, is_time,
data, offset, eieio_header)
elif eieio_type == EIEIOType.KEY_PAYLOAD_16_BIT:
return _read_16_bit_payload_message(prefix, payload_base, prefix_type,
is_time, data, offset,
eieio_header)
elif eieio_type == EIEIOType.KEY_32_BIT:
return _read_32_bit_message(prefix, payload_base, prefix_type, is_time,
data, offset, eieio_header)
elif eieio_type == EIEIOType.KEY_PAYLOAD_32_BIT:
return _read_32_bit_payload_message(prefix, payload_base, prefix_type,
is_time, data, offset,
eieio_header)
return EIEIODataMessage(eieio_header, data, offset)
def __init__(self, key_prefix, payload_prefix, count=0, data=None,
offset=0):
EIEIOWithoutPayloadDataMessage.__init__(
self, EIEIODataHeader(EIEIOType.KEY_32_BIT,
payload_prefix=payload_prefix,
prefix=key_prefix, count=count),
data, offset)
def read_eieio_data_message(data, offset):
eieio_header = EIEIODataHeader.from_bytestring(data, offset)
offset += eieio_header.size
eieio_type = eieio_header.eieio_type
prefix = eieio_header.prefix
payload_base = eieio_header.payload_base
prefix_type = eieio_header.prefix_type
is_time = eieio_header.is_time
if eieio_type == EIEIOType.KEY_16_BIT:
return _read_16_bit_message(
prefix, payload_base, prefix_type, is_time, data, offset,
eieio_header)
elif eieio_type == EIEIOType.KEY_PAYLOAD_16_BIT:
return _read_16_bit_payload_message(
prefix, payload_base, prefix_type, is_time, data, offset,
eieio_header)
elif eieio_type == EIEIOType.KEY_32_BIT:
return _read_32_bit_message(
prefix, payload_base, prefix_type, is_time, data, offset,
eieio_header)
elif eieio_type == EIEIOType.KEY_PAYLOAD_32_BIT:
return _read_32_bit_payload_message(
prefix, payload_base, prefix_type, is_time, data, offset,
eieio_header)
return EIEIODataMessage(eieio_header, data, offset)
def __init__(self, key_prefix, timestamp, count=0, data=None, offset=0):
EIEIOWithoutPayloadDataMessage.__init__(
self,
EIEIODataHeader(EIEIOType.KEY_32_BIT,
payload_base=timestamp,
is_time=True,
prefix=key_prefix,
count=count), data, offset)
def get_spikes(self, label, buffer_manager, region, placements,
graph_mapper, application_vertex, base_key_function,
machine_time_step):
results = list()
missing_str = ""
ms_per_tick = machine_time_step / 1000.0
vertices = \
graph_mapper.get_machine_vertices(application_vertex)
progress_bar = ProgressBar(len(vertices),
"Getting spikes for {}".format(label))
for vertex in vertices:
placement = placements.get_placement_of_vertex(vertex)
vertex_slice = graph_mapper.get_slice(vertex)
x = placement.x
y = placement.y
p = placement.p
# Read the spikes
raw_spike_data, data_missing = \
buffer_manager.get_data_for_vertex(placement, region)
if data_missing:
missing_str += "({}, {}, {}); ".format(x, y, p)
spike_data = str(raw_spike_data.read_all())
number_of_bytes_written = len(spike_data)
offset = 0
while offset < number_of_bytes_written:
eieio_header = EIEIODataHeader.from_bytestring(
spike_data, offset)
offset += eieio_header.size
timestamp = eieio_header.payload_base * ms_per_tick
timestamps = numpy.repeat([timestamp], eieio_header.count)
keys = numpy.frombuffer(spike_data,
dtype="<u4",
count=eieio_header.count,
offset=offset)
neuron_ids = ((keys - base_key_function(vertex)) +
vertex_slice.lo_atom)
offset += eieio_header.count * 4
results.append(numpy.dstack((neuron_ids, timestamps))[0])
progress_bar.update()
progress_bar.end()
if len(missing_str) > 0:
logger.warn(
"Population {} is missing spike data in region {} from the"
" following cores: {}".format(label, region, missing_str))
if len(results) != 0:
result = numpy.vstack(results)
result = result[numpy.lexsort((result[:, 1], result[:, 0]))]
else:
result = []
return result
def get_spikes(self, label, buffer_manager, region, state_region,
placements, graph_mapper, partitionable_vertex,
base_key_function):
results = list()
missing_str = ""
ms_per_tick = self._machine_time_step / 1000.0
subvertices = \
graph_mapper.get_subvertices_from_vertex(partitionable_vertex)
progress_bar = ProgressBar(len(subvertices),
"Getting spikes for {}".format(label))
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
x = placement.x
y = placement.y
p = placement.p
# Read the spikes
raw_spike_data, data_missing = \
buffer_manager.get_data_for_vertex(
x, y, p, region, state_region)
if data_missing:
missing_str += "({}, {}, {}); ".format(x, y, p)
spike_data = raw_spike_data.read_all()
number_of_bytes_written = len(spike_data)
offset = 0
while offset < number_of_bytes_written:
eieio_header = EIEIODataHeader.from_bytestring(
spike_data, offset)
offset += eieio_header.size
timestamp = eieio_header.payload_base * ms_per_tick
timestamps = numpy.repeat([timestamp], eieio_header.count)
keys = numpy.frombuffer(
spike_data, dtype="<u4", count=eieio_header.count,
offset=offset)
neuron_ids = ((keys - base_key_function(subvertex)) +
subvertex_slice.lo_atom)
offset += eieio_header.count * 4
results.append(numpy.dstack((neuron_ids, timestamps))[0])
progress_bar.update()
progress_bar.end()
if len(missing_str) > 0:
logger.warn(
"Population {} is missing spike data in region {} from the"
" following cores: {}".format(label, region, missing_str))
if len(results) != 0:
result = numpy.vstack(results)
result = result[numpy.lexsort((result[:, 1], result[:, 0]))]
else:
result = []
return result
def __init__(self, key_prefix, count=0, data=None, offset=0):
EIEIOWithoutPayloadDataMessage.__init__(
self,
EIEIODataHeader(EIEIOType.KEY_16_BIT,
is_time=True,
prefix=key_prefix,
prefix_type=EIEIOPrefix.UPPER_HALF_WORD,
count=count), data, offset)
def min_packet_length(eieio_type, is_prefix=False, is_payload_base=False):
""" The minimum length of a message with the given header, in bytes
:param eieio_type: the type of message
:type eieio_type:\
:py:class:`spinnman.spinnman.messages.eieio.eieio_type.EIEIOType`
:param is_prefix: True if there is a prefix, False otherwise
:type is_prefix: bool
:param is_payload_base: True if there is a payload base, False\
otherwise
:type is_payload_base: bool
:return: The minimum size of the packet in bytes
:rtype: int
"""
header_size = EIEIODataHeader.get_header_size(eieio_type, is_prefix,
is_payload_base)
return header_size + eieio_type.payload_bytes
def min_packet_length(eieio_type, is_prefix=False, is_payload_base=False):
""" The minimum length of a message with the given header, in bytes
:param eieio_type: the type of message
:type eieio_type:\
:py:class:`spinnman.spinnman.messages.eieio.eieio_type.EIEIOType`
:param is_prefix: True if there is a prefix, False otherwise
:type is_prefix: bool
:param is_payload_base: True if there is a payload base, False\
otherwise
:type is_payload_base: bool
:return: The minimum size of the packet in bytes
:rtype: int
"""
header_size = EIEIODataHeader.get_header_size(eieio_type, is_prefix,
is_payload_base)
return header_size + eieio_type.payload_bytes
def get_spikes(self, label, transceiver, region, placements, graph_mapper,
partitionable_vertex):
results = list()
ms_per_tick = self._machine_time_step / 1000.0
subvertices = \
graph_mapper.get_subvertices_from_vertex(partitionable_vertex)
progress_bar = ProgressBar(len(subvertices),
"Getting spikes for {}".format(label))
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
# Read the spikes
spike_data = recording_utils.get_data(
transceiver, placement, region, subvertex.region_size)
number_of_bytes_written = len(spike_data)
offset = 0
while offset < number_of_bytes_written:
eieio_header = EIEIODataHeader.from_bytestring(
spike_data, offset)
offset += eieio_header.size
timestamp = eieio_header.payload_base * ms_per_tick
timestamps = numpy.repeat([timestamp], eieio_header.count)
keys = numpy.frombuffer(
spike_data, dtype="<u4", count=eieio_header.count,
offset=offset)
neuron_ids = \
(keys - subvertex.base_key) + subvertex_slice.lo_atom
offset += eieio_header.count * 4
results.append(numpy.dstack((neuron_ids, timestamps))[0])
progress_bar.update()
progress_bar.end()
result = numpy.vstack(results)
result = result[numpy.lexsort((result[:, 1], result[:, 0]))]
return result
def __init__(self, payload_prefix, count=0, data=None, offset=0):
EIEIOWithPayloadDataMessage.__init__(
self, EIEIODataHeader(EIEIOType.KEY_PAYLOAD_16_BIT,
payload_base=payload_prefix, count=count),
data, offset)
def __init__(self, count=0, data=None, offset=0):
EIEIOWithPayloadDataMessage.__init__(
self, EIEIODataHeader(EIEIOType.KEY_PAYLOAD_32_BIT, count=count),
data, offset)
import StopRequests
from spynnaker.pyNN.exceptions import BufferableRegionTooSmall
from spynnaker.pyNN.utilities import utility_calls
from spynnaker.pyNN.buffer_management.storage_objects.buffers_sent_deque\
import BuffersSentDeque
logger = logging.getLogger(__name__)
# The minimum size of any message - this is the headers plus one entry
_MIN_MESSAGE_SIZE = (EIEIO32BitTimedPayloadPrefixDataMessage
.get_min_packet_length())
# The size of the header of a message
_HEADER_SIZE = EIEIODataHeader.get_header_size(EIEIOType.KEY_32_BIT,
is_payload_base=True)
# The number of bytes in each key to be sent
_N_BYTES_PER_KEY = EIEIOType.KEY_32_BIT.key_bytes
# The number of keys allowed (different from the actual number as there is an
# additional header)
_N_KEYS_PER_MESSAGE = (constants.UDP_MESSAGE_MAX_SIZE -
(HostSendSequencedData.get_min_packet_length() +
_HEADER_SIZE) / _N_BYTES_PER_KEY)
class BufferManager(object):
""" Manager of send buffers
"""
class BufferedSendingRegion(object):
""" A set of keys to be sent at given timestamps for a given region of\
data. Note that keys must be added in timestamp order or else an\
exception will be raised
"""
_HEADER_SIZE = EIEIODataHeader.get_header_size(EIEIOType.KEY_32_BIT,
is_payload_base=True)
# The number of bytes in each key to be sent
_N_BYTES_PER_KEY = EIEIOType.KEY_32_BIT.key_bytes # @UndefinedVariable
# The number of keys allowed (different from the actual number as there is
# an additional header)
_N_KEYS_PER_MESSAGE = (constants.UDP_MESSAGE_MAX_SIZE -
(HostSendSequencedData.get_min_packet_length() +
_HEADER_SIZE)) / _N_BYTES_PER_KEY
def __init__(self, max_buffer_size):
self._max_size_of_buffer = max_buffer_size
self.clear()
@property
def buffer_size(self):
"""
property method for getting the max size of this buffer
:return:
"""
if self._buffer_size is None:
self._calculate_sizes()
return self._buffer_size
@property
def total_region_size(self):
""" Get the max size of this region
:return:
"""
if self._total_region_size is None:
self._calculate_sizes()
return self._total_region_size
@property
def max_buffer_size_possible(self):
""" Get the max possible size of a buffer from this region
:return:
"""
return self._max_size_of_buffer
def _calculate_sizes(self):
""" Deduce how big the buffer and the region needs to be
:return:
"""
size = 0
for timestamp in self._timestamps:
n_keys = self.get_n_keys(timestamp)
size += self.get_n_bytes(n_keys)
size += EventStopRequest.get_min_packet_length()
if size > self._max_size_of_buffer:
self._buffer_size = self._max_size_of_buffer
else:
self._buffer_size = size
self._total_region_size = size
def get_n_bytes(self, n_keys):
""" Get the number of bytes used by a given number of keys
:param n_keys: The number of keys
:type n_keys: int
"""
# Get the total number of messages
n_messages = int(math.ceil(float(n_keys) / self._N_KEYS_PER_MESSAGE))
# Add up the bytes
return ((self._HEADER_SIZE * n_messages) +
(n_keys * self._N_BYTES_PER_KEY))
def add_key(self, timestamp, key):
""" Add a key to be sent at a given time
:param timestamp: The time at which the key is to be sent
:type timestamp: int
:param key: The key to send
:type key: int
"""
if timestamp not in self._buffer:
bisect.insort(self._timestamps, timestamp)
self._buffer[timestamp] = list()
self._buffer[timestamp].append(key)
self._total_region_size = None
self._buffer_size = None
def add_keys(self, timestamp, keys):
""" Add a set of keys to be sent at the given time
:param timestamp: The time at which the keys are to be sent
:type timestamp: int
:param keys: The keys to send
:type keys: iterable of int
"""
for key in keys:
self.add_key(timestamp, key)
@property
def n_timestamps(self):
""" The number of timestamps available
:rtype: int
"""
return len(self._timestamps)
@property
def timestamps(self):
""" The timestamps for which there are keys
:rtype: iterable of int
"""
return self._timestamps
def get_n_keys(self, timestamp):
""" Get the number of keys for a given timestamp
:param timestamp: the time stamp to check if there's still keys to\
transmit
"""
if timestamp in self._buffer:
return len(self._buffer[timestamp])
return 0
@property
def is_next_timestamp(self):
""" Determines if the region is empty
:return: True if the region is empty, false otherwise
:rtype: bool
"""
return self._current_timestamp_pos < len(self._timestamps)
@property
def next_timestamp(self):
""" The next timestamp of the data to be sent, or None if no more data
:rtype: int or None
"""
if self.is_next_timestamp:
return self._timestamps[self._current_timestamp_pos]
return None
def is_next_key(self, timestamp):
""" Determine if there is another key for the given timestamp
:param timestamp: the time stamp to check if there's still keys to\
transmit
:rtype: bool
"""
if timestamp in self._buffer:
return len(self._buffer[timestamp]) > 0
return False
@property
def next_key(self):
""" The next key to be sent
:rtype: int
"""
next_timestamp = self.next_timestamp
keys = self._buffer[next_timestamp]
key = keys.pop()
if len(keys) == 0:
del self._buffer[next_timestamp]
self._current_timestamp_pos += 1
return key
@property
def current_timestamp(self):
""" Get the current timestamp in the iterator
"""
return self._current_timestamp_pos
def rewind(self):
""" Rewind the buffer to initial position.
"""
self._current_timestamp_pos = 0
def clear(self):
""" Clears the buffer
"""
# A dictionary of timestamp -> list of keys
self._buffer = dict()
# A list of timestamps
self._timestamps = list()
# The current position in the list of timestamps
self._current_timestamp_pos = 0
self._buffer_size = None
self._total_region_size = None
def get_spikes(
self, txrx, placements, graph_mapper, compatible_output=False):
"""
Return a 2-column numpy array containing cell ids and spike times for
recorded cells. This is read directly from the memory for the board.
:param transceiver:
:param placements:
:param graph_mapper:
:param compatible_output:
"""
logger.info("Getting spikes for {}".format(self._label))
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(len(subvertices), "Getting spikes")
results = list()
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placement.x, placement.y, placement.p
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
lo_atom = subvertex_slice.lo_atom
hi_atom = subvertex_slice.hi_atom
logger.debug("Reading spikes from chip {}, {}, core {}, "
"lo_atom {} hi_atom {}".format(
x, y, p, lo_atom, hi_atom))
# Get the App Data for the core
app_data_base_address = \
txrx.get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the spike buffer
spike_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address,
self._SPIKE_SOURCE_REGIONS
.SPIKE_DATA_RECORDED_REGION.value)
spike_region_base_address_buf = buffer(txrx.read_memory(
x, y, spike_region_base_address_offset, 4))
spike_region_base_address = struct.unpack_from(
"<I", spike_region_base_address_buf)[0]
spike_region_base_address += app_data_base_address
# Read the spike data size
number_of_bytes_written_buf = buffer(txrx.read_memory(
x, y, spike_region_base_address, 4))
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# check that the number of spikes written is smaller or the same as
# the size of the memory region we allocated for spikes
send_buffer = self._get_spike_send_buffer(subvertex_slice)
if number_of_bytes_written > send_buffer.total_region_size:
raise exceptions.MemReadException(
"the amount of memory written ({}) was larger than was "
"allocated for it ({})"
.format(number_of_bytes_written,
send_buffer.total_region_size))
# Read the spikes
logger.debug("Reading {} ({}) bytes starting at {} + 4"
.format(number_of_bytes_written,
hex(number_of_bytes_written),
hex(spike_region_base_address)))
spike_data_block = txrx.read_memory(
x, y, spike_region_base_address + 4, number_of_bytes_written)
# translate block of spikes into EIEIO messages
offset = 0
while offset <= number_of_bytes_written - 4:
eieio_header = EIEIODataHeader.from_bytestring(
spike_data_block, offset)
offset += eieio_header.size
timestamps = numpy.repeat([eieio_header.payload_base],
eieio_header.count)
keys = numpy.frombuffer(
spike_data_block, dtype="<u4", count=eieio_header.count,
offset=offset)
neuron_ids = ((keys - subvertex.base_key) +
subvertex_slice.lo_atom)
offset += eieio_header.count * 4
results.append(numpy.dstack((neuron_ids, timestamps))[0])
# complete the buffer
progress_bar.update()
progress_bar.end()
result = numpy.vstack(results)
result = result[numpy.lexsort((result[:, 1], result[:, 0]))]
return result
