def _regenerate_data_spec_for_vertices(self, placement, vertex):
"""
:param ~.Placement placement:
:param ~.AbstractVertex vertex:
:rtype: bool
"""
# If the vertex doesn't regenerate, skip
if not isinstance(vertex, AbstractRewritesDataSpecification):
return False
# If the vertex doesn't require regeneration, skip
if not vertex.requires_memory_regions_to_be_reloaded():
return True
# build the writers for the reports and data
spec_file, spec = get_data_spec_and_file_writer_filename(
placement.x, placement.y, placement.p, self._host, self._rpt_dir,
self._write_text, self._data_dir)
# Execute the regeneration
vertex.regenerate_data_specification(spec, placement)
# execute the spec
with FileDataReader(spec_file) as spec_reader:
data_spec_executor = DataSpecificationExecutor(
spec_reader, SDRAM.max_sdram_found)
data_spec_executor.execute()
try:
os.remove(spec_file)
except Exception: # pylint: disable=broad-except
# Ignore the deletion of files as non-critical
pass
# Read the region table for the placement
regions_base_address = self._txrx.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
start_region = get_region_base_address_offset(regions_base_address, 0)
table_size = get_region_base_address_offset(
regions_base_address, MAX_MEM_REGIONS) - start_region
offsets = REGION_STRUCT.unpack_from(
self._txrx.read_memory(placement.x, placement.y, start_region,
table_size))
# Write the regions to the machine
for i, region in enumerate(data_spec_executor.dsef.mem_regions):
if region is not None and not region.unfilled:
self._txrx.write_memory(
placement.x, placement.y, offsets[i],
region.region_data[:region.max_write_pointer])
return True
def _regenerate_data_spec_for_vertices(transceiver, placement, vertex,
hostname,
reloaded_dsg_report_files_file_path,
write_text_specs,
reloaded_dsg_data_files_file_path):
# pylint: disable=too-many-arguments, too-many-locals
# If the vertex doesn't regenerate, skip
if not isinstance(vertex, AbstractRewritesDataSpecification):
return False
# If the vertex doesn't require regeneration, skip
if not vertex.requires_memory_regions_to_be_reloaded():
return True
# build the writers for the reports and data
spec_file, spec = utility_calls.get_data_spec_and_file_writer_filename(
placement.x, placement.y, placement.p, hostname,
reloaded_dsg_report_files_file_path, write_text_specs,
reloaded_dsg_data_files_file_path)
# Execute the regeneration
vertex.regenerate_data_specification(spec, placement)
# execute the spec
spec_reader = FileDataReader(spec_file)
data_spec_executor = DataSpecificationExecutor(
spec_reader, SDRAM.DEFAULT_SDRAM_BYTES)
data_spec_executor.execute()
# Read the region table for the placement
regions_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
start_region = utility_calls.get_region_base_address_offset(
regions_base_address, 0)
table_size = utility_calls.get_region_base_address_offset(
regions_base_address, MAX_MEM_REGIONS) - start_region
offsets = struct.unpack_from(
"<{}I".format(MAX_MEM_REGIONS),
transceiver.read_memory(placement.x, placement.y, start_region,
table_size))
# Write the regions to the machine
for i, region in enumerate(data_spec_executor.dsef.mem_regions):
if region is not None and not region.unfilled:
transceiver.write_memory(
placement.x, placement.y, offsets[i],
region.region_data[:region.max_write_pointer])
return True
def get_data(transceiver, placement, region, region_size):
""" Get the recorded data from a region
"""
(x, y, p) = placement.x, placement.y, placement.p
app_data_base_address = transceiver.get_cpu_information_from_core(
x, y, p).user[0]
region_base_address_offset = utility_calls.get_region_base_address_offset(
app_data_base_address, region)
region_base_address_buf = buffer(transceiver.read_memory(
x, y, region_base_address_offset, 4))
region_base_address = struct.unpack_from("<I", region_base_address_buf)[0]
region_base_address += app_data_base_address
number_of_bytes_written_buf = buffer(transceiver.read_memory(
x, y, region_base_address, 4))
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# Subtract 4 for the word representing the size itself
expected_size = region_size - _RECORDING_COUNT_SIZE
if number_of_bytes_written > expected_size:
raise exceptions.MemReadException(
"Expected {} bytes but read {}".format(
expected_size, number_of_bytes_written))
return transceiver.read_memory(
x, y, region_base_address + 4, number_of_bytes_written)
def write_provenance_data_in_xml(self, file_path, transceiver,
placement=None):
""" Extracts provenance data from the SDRAM of the core and stores it\
in an xml file for end user digestion
:param file_path: the file path to the xml document
:param transceiver: the spinnman interface object
:param placement: the placement object for this subvertex
:return: None
"""
if placement is None:
raise ConfigurationException(
"To acquire provenance data from the live packet gatherer,"
"you must provide a placement object that points to where the "
"live packet gatherer resides on the spinnaker machine")
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
provanence_data_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address,
self._LIVE_DATA_GATHER_REGIONS.PROVANENCE.value)
provanence_data_region_base_address_buf = \
buffer(transceiver.read_memory(
placement.x, placement.y,
provanence_data_region_base_address_offset, 4))
provanence_data_region_base_address = \
struct.unpack("I", provanence_data_region_base_address_buf)[0]
provanence_data_region_base_address += app_data_base_address
# read in the provenance data
provanence_data_region_contents_buff = \
buffer(transceiver.read_memory(
placement.x, placement.y, provanence_data_region_base_address,
self._PROVANENCE_REGION_SIZE))
provanence_data_region_contents = \
struct.unpack("<II", provanence_data_region_contents_buff)
# create provenance data xml form
from lxml import etree
root = etree.Element("Live_packet_gatherer_located_at_{}_{}_{}"
.format(placement.x, placement.y, placement.p))
none_payload_provanence_data = \
etree.SubElement(root, "lost_packets_without_payload")
payload_provanence_data = \
etree.SubElement(root, "lost_packets_with_payload")
none_payload_provanence_data.text = \
str(provanence_data_region_contents[0])
payload_provanence_data.text = \
str(provanence_data_region_contents[1])
# write xml form into file provided
writer = open(file_path, "w")
writer.write(etree.tostring(root, pretty_print=True))
writer.flush()
writer.close()
def get_data_region_address(transceiver, placement, region):
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
address_location = get_region_base_address_offset(app_data_base_address,
region.value)
return transceiver.read_word(placement.x, placement.y, address_location)
def test_with_application_vertices(self):
""" Test that an application vertex's data is rewritten correctly
"""
# Create a default SDRAM to set the max to default
SDRAM()
reload_region_data = [(0, [0] * 10), (1, [1] * 20)]
vertex = _TestApplicationVertex(10, reload_region_data)
m_slice_1 = Slice(0, 4)
m_slice_2 = Slice(5, 9)
m_vertex_1 = vertex.create_machine_vertex(m_slice_1, None, None, None)
m_vertex_2 = vertex.create_machine_vertex(m_slice_2, None, None, None)
graph_mapper = GraphMapper()
graph_mapper.add_vertex_mapping(m_vertex_1, m_slice_1, vertex)
graph_mapper.add_vertex_mapping(m_vertex_2, m_slice_2, vertex)
placements = Placements(
[Placement(m_vertex_1, 0, 0, 1),
Placement(m_vertex_2, 0, 0, 2)])
user_0_addresses = {
(placement.x, placement.y, placement.p): i * 1000
for i, placement in enumerate(placements.placements)
}
region_addresses = [i for i in range(MAX_MEM_REGIONS)]
transceiver = _MockTransceiver(user_0_addresses, region_addresses)
reloader = DSGRegionReloader()
reloader.__call__(transceiver, placements, "localhost", "test", False,
"test", graph_mapper)
regions_rewritten = transceiver.regions_rewritten
# Check that the number of times the data has been regenerated is
# correct
self.assertEqual(vertex.regenerate_call_count, placements.n_placements)
# Check that the number of regions rewritten is correct
self.assertEqual(len(transceiver.regions_rewritten),
placements.n_placements * len(reload_region_data))
# Check that the data rewritten is correct
for i, placement in enumerate(placements.placements):
user_0_address = user_0_addresses[placement.x, placement.y,
placement.p]
for j in range(len(reload_region_data)):
pos = (i * len(reload_region_data)) + j
region, data = reload_region_data[j]
address = get_region_base_address_offset(
user_0_address, 0) + region_addresses[region]
data = bytearray(numpy.array(data, dtype="uint32").tobytes())
# Check that the base address and data written is correct
self.assertEqual(regions_rewritten[pos], (address, data))
# Delete data files
shutil.rmtree("test")
def _get_data_region_address(transceiver, placement):
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
base_address_offset = get_region_base_address_offset(
app_data_base_address, SDRAMWriter.DATA_REGIONS.DATA.value)
return _ONE_WORD.unpack(
transceiver.read_memory(placement.x, placement.y,
base_address_offset, _ONE_WORD.size))[0]
def _regenerate_data_spec_for_vertices(self, placement, vertex):
# If the vertex doesn't regenerate, skip
if not isinstance(vertex, AbstractRewritesDataSpecification):
return False
# If the vertex doesn't require regeneration, skip
if not vertex.requires_memory_regions_to_be_reloaded():
return True
# build the writers for the reports and data
spec_file, spec = get_data_spec_and_file_writer_filename(
placement.x, placement.y, placement.p, self._hostname,
self._report_dir, self._write_text, self._app_data_dir)
# Execute the regeneration
vertex.regenerate_data_specification(spec, placement)
# execute the spec
spec_reader = FileDataReader(spec_file)
data_spec_executor = DataSpecificationExecutor(
spec_reader, SDRAM.DEFAULT_SDRAM_BYTES)
data_spec_executor.execute()
# Read the region table for the placement
regions_base_address = self._txrx.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
start_region = get_region_base_address_offset(regions_base_address, 0)
table_size = get_region_base_address_offset(
regions_base_address, MAX_MEM_REGIONS) - start_region
offsets = REGION_STRUCT.unpack_from(
self._txrx.read_memory(
placement.x, placement.y, start_region, table_size))
# Write the regions to the machine
for i, region in enumerate(data_spec_executor.dsef.mem_regions):
if region is not None and not region.unfilled:
self._txrx.write_memory(
placement.x, placement.y, offsets[i],
region.region_data[:region.max_write_pointer])
return True
def _get_provenance_region_address(self, transceiver, placement):
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
base_address_offset = get_region_base_address_offset(
app_data_base_address, self._provenance_region_id)
base_address = transceiver.read_memory(placement.x, placement.y,
base_address_offset, 4)
return _ONE_WORD.unpack(base_address)[0]
def _get_provenance_region_address(self, transceiver, placement):
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
base_address_offset = get_region_base_address_offset(
app_data_base_address, self._provenance_region_id)
base_address = transceiver.read_memory(
placement.x, placement.y, base_address_offset, 4)
return _ONE_WORD.unpack(base_address)[0]
def _get_provenance_region_address(self, transceiver, placement):
# Get the App Data for the core
app_data_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the provenance region base address
provenance_data_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, self._provenance_region_id)
provenance_data_region_base_address_buff = buffer(
transceiver.read_memory(
placement.x, placement.y,
provenance_data_region_base_address_offset, 4))
provenance_data_region_base_address = struct.unpack(
"<I", provenance_data_region_base_address_buff)[0]
return provenance_data_region_base_address
def locate_master_pop_table_base_address(self, x, y, p, transceiver,
master_pop_table_region):
"""
:param x: x coord for the chip to whcih this master pop table is \
being read
:type x: int
:param y: y coord for the chip to whcih this master pop table is \
being read
:type y: int
:param p: p coord for the processor to whcih this master pop table is \
being read
:type p: int
:param transceiver: the transciever object
:type transceiver: spinnman.transciever.Transciever object
:param master_pop_table_region: the region to which the master pop\
resides
:type master_pop_table_region: int
:return: the master pop table in some form
"""
# Get the App Data base address for the core
# (location where this cores memory starts in
# sdram and region table)
app_data_base_address = \
transceiver.get_cpu_information_from_core(x, y, p).user[0]
# Get the memory address of the master pop table region
master_pop_region = master_pop_table_region
master_region_base_address_address = \
dsg_utility.get_region_base_address_offset(
app_data_base_address, master_pop_region)
master_region_base_address_offset = \
self.read_and_convert(x, y, master_region_base_address_address,
4, "<I", transceiver)
master_region_base_address =\
master_region_base_address_offset + app_data_base_address
return master_region_base_address, app_data_base_address
def locate_master_pop_table_base_address(self, x, y, p, transceiver,
master_pop_table_region):
"""
:param x: x coord for the chip to whcih this master pop table is \
being read
:type x: int
:param y: y coord for the chip to whcih this master pop table is \
being read
:type y: int
:param p: p coord for the processor to whcih this master pop table is \
being read
:type p: int
:param transceiver: the transciever object
:type transceiver: spinnman.transciever.Transciever object
:param master_pop_table_region: the region to which the master pop\
resides
:type master_pop_table_region: int
:return: the master pop table in some form
"""
# Get the App Data base address for the core
# (location where this cores memory starts in
# sdram and region table)
app_data_base_address = \
transceiver.get_cpu_information_from_core(x, y, p).user[0]
# Get the memory address of the master pop table region
master_pop_region = master_pop_table_region
master_region_base_address_address = \
dsg_utility.get_region_base_address_offset(
app_data_base_address, master_pop_region)
master_region_base_address_offset = helpful_functions.read_data(
x, y, master_region_base_address_address, 4, "<I", transceiver)
master_region_base_address =\
master_region_base_address_offset + app_data_base_address
return master_region_base_address, app_data_base_address
def locate_memory_region_for_placement(placement, region, transceiver):
""" Get the address of a region for a placement
:param region: the region to locate the base address of
:type region: int
:param placement: the placement object to get the region address of
:type placement: pacman.model.placements.Placement
:param transceiver: the python interface to the SpiNNaker machine
:type transceiver: spiNNMan.transciever.Transciever
"""
regions_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the position of the region in the pointer table
region_offset = utility_calls.get_region_base_address_offset(
regions_base_address, region)
# Get the actual address of the region
region_address = transceiver.read_memory(
placement.x, placement.y, region_offset, 4)
return _ONE_WORD.unpack_from(region_address)[0]
def locate_memory_region_for_placement(placement, region, transceiver):
""" Get the address of a region for a placement
:param region: the region to locate the base address of
:type region: int
:param placement: the placement object to get the region address of
:type placement: pacman.model.placements.Placement
:param transceiver: the python interface to the SpiNNaker machine
:type transceiver: spiNNMan.transciever.Transciever
"""
regions_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the position of the region in the pointer table
region_offset = utility_calls.get_region_base_address_offset(
regions_base_address, region)
# Get the actual address of the region
region_address = transceiver.read_memory(
placement.x, placement.y, region_offset, 4)
return _ONE_WORD.unpack_from(region_address)[0]
def _locate_region_address(self, region, vertex):
""" Get the address of a region for a vertex
:param region: the region to locate the base address of
:type region: int
:param vertex: the vertex to load a buffer for
:type vertex:\
:py:class:`spynnaker.pyNN.models.abstract_models.buffer_models.abstract_sends_buffers_from_host_partitioned_vertex.AbstractSendsBuffersFromHostPartitionedVertex`
:return: None
"""
placement = self._placements.get_placement_of_subvertex(vertex)
app_data_base_address = \
self._transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the position of the region in the pointer table
region_offset_in_pointer_table = \
dsg_utilities.get_region_base_address_offset(
app_data_base_address, region)
region_offset = str(list(self._transceiver.read_memory(
placement.x, placement.y, region_offset_in_pointer_table, 4))[0])
return struct.unpack("<I", region_offset)[0] + app_data_base_address
def locate_memory_region_for_placement(placement, region, transceiver):
""" Get the address of a region for a placement
:param region: the region to locate the base address of
:type region: int
:param placement: the placement object to get the region address of
:type placement: pacman.model.placements.Placement
:param transceiver: the python interface to the spinnaker machine
:type transceiver: spiNNMan.transciever.Transciever
"""
regions_base_address = transceiver.get_cpu_information_from_core(
placement.x, placement.y, placement.p).user[0]
# Get the position of the region in the pointer table
region_offset_in_pointer_table = \
utility_calls.get_region_base_address_offset(
regions_base_address, region)
region_address = buffer(
transceiver.read_memory(placement.x, placement.y,
region_offset_in_pointer_table, 4))
region_address_decoded = struct.unpack_from("<I", region_address)[0]
return region_address_decoded
def _get_spikes(
self, graph_mapper, placements, transceiver, compatible_output,
spike_recording_region, sub_vertex_out_spike_bytes_function):
"""
Return a 2-column numpy array containing cell ids and spike times for
recorded cells. This is read directly from the memory for the board.
"""
spike_times = list()
spike_ids = list()
ms_per_tick = self._machine_time_step / 1000.0
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(
len(subvertices), "Getting spikes for {}".format(self._label))
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placement.x, placement.y, placement.p
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
lo_atom = subvertex_slice.lo_atom
hi_atom = subvertex_slice.hi_atom
logger.debug("Reading spikes from chip {}, {}, core {}, "
"lo_atom {} hi_atom {}".format(
x, y, p, lo_atom, hi_atom))
# Get the App Data for the core
app_data_base_address = \
transceiver.get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the spike buffer
spike_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, spike_recording_region)
spike_region_base_address_buf = buffer(transceiver.read_memory(
x, y, spike_region_base_address_offset, 4))
spike_region_base_address = struct.unpack_from(
"<I", spike_region_base_address_buf)[0]
spike_region_base_address += app_data_base_address
# Read the spike data size
number_of_bytes_written_buf = buffer(transceiver.read_memory(
x, y, spike_region_base_address, 4))
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# check that the number of spikes written is smaller or the same as
# the size of the memory region we allocated for spikes
out_spike_bytes = sub_vertex_out_spike_bytes_function(
subvertex, subvertex_slice)
size_of_region = self.get_recording_region_size(out_spike_bytes)
if number_of_bytes_written > size_of_region:
raise exceptions.MemReadException(
"the amount of memory written ({}) was larger than was "
"allocated for it ({})"
.format(number_of_bytes_written, size_of_region))
# Read the spikes
logger.debug("Reading {} ({}) bytes starting at {} + 4"
.format(number_of_bytes_written,
hex(number_of_bytes_written),
hex(spike_region_base_address)))
spike_data = transceiver.read_memory(
x, y, spike_region_base_address + 4, number_of_bytes_written)
numpy_data = numpy.asarray(spike_data, dtype="uint8").view(
dtype="uint32").byteswap().view("uint8")
bits = numpy.fliplr(numpy.unpackbits(numpy_data).reshape(
(-1, 32))).reshape((-1, out_spike_bytes * 8))
times, indices = numpy.where(bits == 1)
times = times * ms_per_tick
indices = indices + lo_atom
spike_ids.append(indices)
spike_times.append(times)
progress_bar.update()
progress_bar.end()
spike_ids = numpy.hstack(spike_ids)
spike_times = numpy.hstack(spike_times)
result = numpy.dstack((spike_ids, spike_times))[0]
return result[numpy.lexsort((spike_times, spike_ids))]
def _get_gsyn(
self, region, compatible_output, has_ran, graph_mapper, placements,
txrx, machine_time_step, runtime):
if not has_ran:
raise exceptions.SpynnakerException(
"The simulation has not yet ran, therefore neuron param "
"cannot be retrieved")
ms_per_tick = self._machine_time_step / 1000.0
n_timesteps = runtime / ms_per_tick
tempfilehandle = tempfile.NamedTemporaryFile()
data = numpy.memmap(
tempfilehandle.file, shape=(n_timesteps, self._n_atoms),
dtype="float64,float64,float64,float64")
data["f0"] = (numpy.arange(self._n_atoms * n_timesteps) %
self._n_atoms).reshape((n_timesteps, self._n_atoms))
data["f1"] = numpy.repeat(numpy.arange(0, n_timesteps * ms_per_tick,
ms_per_tick), self._n_atoms).reshape(
(n_timesteps, self._n_atoms))
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(
len(subvertices), "Getting recorded gsyn for {}".format(
self._label))
for subvertex in subvertices:
placment = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placment.x, placment.y, placment.p
# Get the App Data for the core
app_data_base_address = txrx.\
get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the value buffer
neuron_param_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, region)
neuron_param_region_base_address_buf = buffer(txrx.read_memory(
x, y, neuron_param_region_base_address_offset, 4))
neuron_param_region_base_address = struct.unpack_from(
"<I", neuron_param_region_base_address_buf)[0]
neuron_param_region_base_address += app_data_base_address
# Read the size
number_of_bytes_written_buf = buffer(txrx.read_memory(
x, y, neuron_param_region_base_address, 4))
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# Read the values
logger.debug("Reading {} ({}) bytes starting at {}".format(
number_of_bytes_written, hex(number_of_bytes_written),
hex(neuron_param_region_base_address + 4)))
neuron_param_region_data = txrx.read_memory(
x, y, neuron_param_region_base_address + 4,
number_of_bytes_written)
vertex_slice = graph_mapper.get_subvertex_slice(subvertex)
bytes_per_time_step = vertex_slice.n_atoms * 4
number_of_time_steps_written = \
number_of_bytes_written / bytes_per_time_step
logger.debug("Processing {} timesteps"
.format(number_of_time_steps_written))
numpy_data = (numpy.asarray(
neuron_param_region_data, dtype="uint8").view(dtype="<i4") /
32767.0).reshape((n_timesteps, vertex_slice.n_atoms * 2))
data["f2"][:, vertex_slice.lo_atom:vertex_slice.hi_atom + 1] =\
numpy_data[:, 0::2]
data["f3"][:, vertex_slice.lo_atom:vertex_slice.hi_atom + 1] =\
numpy_data[:, 1::2]
progress_bar.update()
progress_bar.end()
data.shape = self._n_atoms * n_timesteps
# Sort the data - apparently, using lexsort is faster, but it might
# consume more memory, so the option is left open for sort-in-place
order = numpy.lexsort((data["f1"], data["f0"]))
# data.sort(order=['f0', 'f1'], axis=0)
result = data.view(dtype="float64").reshape(
(self._n_atoms * n_timesteps, 4))[order]
return result
def get_neuron_parameter(
self, region, compatible_output, has_ran, graph_mapper, placements,
txrx, machine_time_step):
if not has_ran:
raise exceptions.SpynnakerException(
"The simulation has not yet ran, therefore neuron param "
"cannot be retrieved")
times = numpy.zeros(0)
ids = numpy.zeros(0)
values = numpy.zeros(0)
ms_per_tick = self._machine_time_step / 1000.0
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(len(subvertices), "Getting recorded data")
for subvertex in subvertices:
placment = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placment.x, placment.y, placment.p
# Get the App Data for the core
app_data_base_address = txrx.\
get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the value buffer
neuron_param_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, region)
neuron_param_region_base_address_buf = str(list(txrx.read_memory(
x, y, neuron_param_region_base_address_offset, 4))[0])
neuron_param_region_base_address = \
struct.unpack("<I", neuron_param_region_base_address_buf)[0]
neuron_param_region_base_address += app_data_base_address
# Read the size
number_of_bytes_written_buf = \
str(list(txrx.read_memory(
x, y, neuron_param_region_base_address, 4))[0])
number_of_bytes_written = \
struct.unpack_from("<I", number_of_bytes_written_buf)[0]
# Read the values
logger.debug("Reading {} ({}) bytes starting at {}".format(
number_of_bytes_written, hex(number_of_bytes_written),
hex(neuron_param_region_base_address + 4)))
neuron_param_region_data = txrx.read_memory(
x, y, neuron_param_region_base_address + 4,
number_of_bytes_written)
vertex_slice = graph_mapper.get_subvertex_slice(subvertex)
n_atoms = (vertex_slice.hi_atom - vertex_slice.lo_atom) + 1
bytes_per_time_step = n_atoms * 4
number_of_time_steps_written = \
number_of_bytes_written / bytes_per_time_step
logger.debug("Processing {} timesteps"
.format(number_of_time_steps_written))
data_list = bytearray()
for data in neuron_param_region_data:
data_list.extend(data)
numpy_data = numpy.asarray(data_list, dtype="uint8").view(
dtype="<i4") / 32767.0
values = numpy.append(values, numpy_data)
times = numpy.append(
times, numpy.repeat(range(numpy_data.size / n_atoms),
n_atoms) * ms_per_tick)
ids = numpy.append(ids, numpy.add(
numpy.arange(numpy_data.size) % n_atoms, vertex_slice.lo_atom))
progress_bar.update()
progress_bar.end()
result = numpy.dstack((ids, times, values))[0]
result = result[numpy.lexsort((times, ids))]
return result
def _retrieve_synaptic_block(self, placements, transceiver, pre_subvertex,
pre_n_atoms, post_subvertex, routing_infos,
subgraph):
"""
reads in a synaptic block from a given processor and subvertex on the
machine.
"""
post_placement = placements.get_placement_of_subvertex(post_subvertex)
post_x, post_y, post_p = \
post_placement.x, post_placement.y, post_placement.p
# either read in the master pop table or retrieve it from storage
master_pop_base_mem_address, app_data_base_address = \
self._master_pop_table_generator.\
locate_master_pop_table_base_address(
post_x, post_y, post_p, transceiver,
constants.POPULATION_BASED_REGIONS.POPULATION_TABLE.value)
incoming_edges = subgraph.incoming_subedges_from_subvertex(
post_subvertex)
incoming_key_combo = None
for subedge in incoming_edges:
if subedge.pre_subvertex == pre_subvertex:
routing_info = \
routing_infos.get_subedge_information_from_subedge(subedge)
keys_and_masks = routing_info.keys_and_masks
incoming_key_combo = keys_and_masks[0].key
break
maxed_row_length, synaptic_block_base_address_offset = \
self._master_pop_table_generator.\
extract_synaptic_matrix_data_location(
incoming_key_combo, master_pop_base_mem_address,
transceiver, post_x, post_y)
block = None
if maxed_row_length > 0:
# calculate the synaptic block size in words
synaptic_block_size = (
pre_n_atoms * 4 *
(constants.SYNAPTIC_ROW_HEADER_WORDS + maxed_row_length))
# read in the base address of the synaptic matrix in the app region
# table
synapse_region_base_address_location = \
dsg_utilities.get_region_base_address_offset(
app_data_base_address,
constants.POPULATION_BASED_REGIONS.SYNAPTIC_MATRIX.value)
# read in the memory address of the synaptic_region base address
synapse_region_base_address = helpful_functions.read_data(
post_x, post_y, synapse_region_base_address_location, 4, "<I",
transceiver)
# the base address of the synaptic block in absolute terms is the app
# base, plus the synaptic matrix base plus the offset
synaptic_block_base_address = (app_data_base_address +
synapse_region_base_address +
synaptic_block_base_address_offset)
# read in and return the synaptic block
block = transceiver.read_memory(post_x, post_y,
synaptic_block_base_address,
synaptic_block_size)
if len(block) != synaptic_block_size:
raise exceptions.SynapticBlockReadException(
"Not enough data has been read"
" (aka, something funkky happened)")
return block, maxed_row_length
def _get_spikes(self, graph_mapper, placements, transceiver,
compatible_output, spike_recording_region,
sub_vertex_out_spike_bytes_function):
"""
Return a 2-column numpy array containing cell ids and spike times for
recorded cells. This is read directly from the memory for the board.
"""
logger.info("Getting spikes for {}".format(self._label))
spike_times = list()
spike_ids = list()
ms_per_tick = self._machine_time_step / 1000.0
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(len(subvertices), "Getting spikes")
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placement.x, placement.y, placement.p
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
lo_atom = subvertex_slice.lo_atom
hi_atom = subvertex_slice.hi_atom
logger.debug("Reading spikes from chip {}, {}, core {}, "
"lo_atom {} hi_atom {}".format(
x, y, p, lo_atom, hi_atom))
# Get the App Data for the core
app_data_base_address = \
transceiver.get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the spike buffer
spike_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, spike_recording_region)
spike_region_base_address_buf = transceiver.read_memory(
x, y, spike_region_base_address_offset, 4)
spike_region_base_address = struct.unpack_from(
"<I", spike_region_base_address_buf)[0]
spike_region_base_address += app_data_base_address
# Read the spike data size
number_of_bytes_written_buf = transceiver.read_memory(
x, y, spike_region_base_address, 4)
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# check that the number of spikes written is smaller or the same as
# the size of the memory region we allocated for spikes
out_spike_bytes = sub_vertex_out_spike_bytes_function(
subvertex, subvertex_slice)
size_of_region = self.get_recording_region_size(out_spike_bytes)
if number_of_bytes_written > size_of_region:
raise exceptions.MemReadException(
"the amount of memory written ({}) was larger than was "
"allocated for it ({})".format(number_of_bytes_written,
size_of_region))
# Read the spikes
logger.debug("Reading {} ({}) bytes starting at {} + 4".format(
number_of_bytes_written, hex(number_of_bytes_written),
hex(spike_region_base_address)))
spike_data = transceiver.read_memory(x, y,
spike_region_base_address + 4,
number_of_bytes_written)
numpy_data = numpy.asarray(
spike_data,
dtype="uint8").view(dtype="uint32").byteswap().view("uint8")
bits = numpy.fliplr(
numpy.unpackbits(numpy_data).reshape((-1, 32))).reshape(
(-1, out_spike_bytes * 8))
times, indices = numpy.where(bits == 1)
times = times * ms_per_tick
indices = indices + lo_atom
spike_ids.append(indices)
spike_times.append(times)
progress_bar.update()
progress_bar.end()
spike_ids = numpy.hstack(spike_ids)
spike_times = numpy.hstack(spike_times)
result = numpy.dstack((spike_ids, spike_times))[0]
return result[numpy.lexsort((spike_times, spike_ids))]
def get_neuron_parameter(self, region, compatible_output, has_ran,
graph_mapper, placements, txrx, machine_time_step,
runtime):
if not has_ran:
raise exceptions.SpynnakerException(
"The simulation has not yet ran, therefore neuron param "
"cannot be retrieved")
ms_per_tick = self._machine_time_step / 1000.0
n_timesteps = runtime / ms_per_tick
tempfilehandle = tempfile.NamedTemporaryFile()
data = numpy.memmap(tempfilehandle.file,
shape=(n_timesteps, self._n_atoms),
dtype="float64,float64,float64")
data["f0"] = (numpy.arange(self._n_atoms * n_timesteps) %
self._n_atoms).reshape((n_timesteps, self._n_atoms))
data["f1"] = numpy.repeat(
numpy.arange(0, n_timesteps * ms_per_tick, ms_per_tick),
self._n_atoms).reshape((n_timesteps, self._n_atoms))
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(len(subvertices), "Getting recorded data")
for subvertex in subvertices:
placment = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placment.x, placment.y, placment.p
# Get the App Data for the core
app_data_base_address = txrx.\
get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the value buffer
neuron_param_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address, region)
neuron_param_region_base_address_buf = txrx.read_memory(
x, y, neuron_param_region_base_address_offset, 4)
neuron_param_region_base_address = struct.unpack_from(
"<I", neuron_param_region_base_address_buf)[0]
neuron_param_region_base_address += app_data_base_address
# Read the size
number_of_bytes_written_buf = txrx.read_memory(
x, y, neuron_param_region_base_address, 4)
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# Read the values
logger.debug("Reading {} ({}) bytes starting at {}".format(
number_of_bytes_written, hex(number_of_bytes_written),
hex(neuron_param_region_base_address + 4)))
neuron_param_region_data = txrx.read_memory(
x, y, neuron_param_region_base_address + 4,
number_of_bytes_written)
vertex_slice = graph_mapper.get_subvertex_slice(subvertex)
bytes_per_time_step = vertex_slice.n_atoms * 4
number_of_time_steps_written = \
number_of_bytes_written / bytes_per_time_step
logger.debug(
"Processing {} timesteps".format(number_of_time_steps_written))
numpy_data = (numpy.asarray(neuron_param_region_data,
dtype="uint8").view(dtype="<i4") /
32767.0).reshape((n_timesteps, vertex_slice.n_atoms))
data["f2"][:, vertex_slice.lo_atom:vertex_slice.hi_atom + 1] =\
numpy_data
progress_bar.update()
progress_bar.end()
data.shape = self._n_atoms * n_timesteps
# Sort the data - apparently, using lexsort is faster, but it might
# consume more memory, so the option is left open for sort-in-place
order = numpy.lexsort((data["f1"], data["f0"]))
# data.sort(order=['f0', 'f1'], axis=0)
result = data.view(dtype="float64").reshape(
(self._n_atoms * n_timesteps, 3))[order]
return result
def test_get_region_base_address_offset(self):
val = get_region_base_address_offset(48, 7)
self.assertEqual(val, 84)
def get_data_for_vertex(self, x, y, p, region_to_read, state_region):
""" Get a pointer to the data container for all the data retrieved\
during the simulation from a specific region area of a core
:param x: x coordinate of the chip
:type x: int
:param y: y coordinate of the chip
:type y: int
:param p: processor on the specified chip
:type p: int
:param region_to_read: desired data region
:type region_to_read: int
:param state_region: final state storage region
:type state_region: int
:return: pointer to a class which inherits from\
AbstractBufferedDataStorage
:rtype:\
py:class:`spinn_front_end_common.interface.buffer_management.buffer_models.abstract_buffered_data_storage.AbstractBufferedDataStorage`
"""
# flush data here
if not self._received_data.is_data_from_region_flushed(
x, y, p, region_to_read):
if not self._received_data.is_end_buffering_state_recovered(
x, y, p):
# Get the App Data for the core
app_data_base_address = \
self._transceiver.get_cpu_information_from_core(
x, y, p).user[0]
# Get the position of the buffer
state_region_base_offset_address = \
dsg_utilities.get_region_base_address_offset(
app_data_base_address, state_region)
state_region_base_address_buf = buffer(
self._transceiver.read_memory(
x, y, state_region_base_offset_address, 4))
state_region_base_address = struct.unpack_from(
"<I", state_region_base_address_buf)[0]
state_region_base_address += app_data_base_address
# retrieve channel state memory area
raw_number_of_channels = self._transceiver.read_memory(
x, y, state_region_base_address, 4)
number_of_channels = struct.unpack(
"<I", str(raw_number_of_channels))[0]
channel_state_data = str(self._transceiver.read_memory(
x, y, state_region_base_address,
EndBufferingState.size_of_region(number_of_channels)))
end_buffering_state = EndBufferingState.create_from_bytearray(
channel_state_data)
self._received_data.store_end_buffering_state(
x, y, p, end_buffering_state)
else:
end_buffering_state = self._received_data.\
get_end_buffering_state(x, y, p)
end_state = end_buffering_state.get_state_for_region(
region_to_read)
start_ptr = end_state.start_address
write_ptr = end_state.current_write
end_ptr = end_state.end_address
read_ptr = end_state.current_read
# current read needs to be adjusted in case the last portion of the
# memory has already been read, but the HostDataRead packet has not
# been processed by the chip before simulation finished
# This situation is identified by the sequence number of the last
# packet sent to this core and the core internal state of the
# output buffering finite state machine
seq_no_last_ack_packet = \
self._received_data.last_sequence_no_for_core(x, y, p)
seq_no_internal_fsm = end_buffering_state.buffering_out_fsm_state
if seq_no_internal_fsm == seq_no_last_ack_packet:
# if the last ack packet has not been processed on the chip,
# process it now
last_sent_ack_sdp_packet = \
self._received_data.last_sent_packet_to_core(x, y, p)
last_sent_ack_packet = create_eieio_command.\
read_eieio_command_message(
last_sent_ack_sdp_packet.data, 0)
if not isinstance(last_sent_ack_packet, HostDataRead):
raise Exception(
"Something somewhere went terribly wrong - "
"I was looking for a HostDataRead packet, "
"while I got {0:s}".format(last_sent_ack_packet))
for i in xrange(last_sent_ack_packet.n_requests):
if (region_to_read == last_sent_ack_packet.region_id(i) and
not end_state.is_state_updated):
read_ptr += last_sent_ack_packet.space_read(i)
if (read_ptr == write_ptr or
(read_ptr == end_ptr and
write_ptr == start_ptr)):
end_state.update_last_operation(
spinn_front_end_constants.BUFFERING_OPERATIONS.
BUFFER_READ.value)
if read_ptr == end_ptr:
read_ptr = start_ptr
elif read_ptr > end_ptr:
raise Exception(
"Something somewhere went terribly wrong - "
"I was reading beyond the region area some "
"unknown data".format(
last_sent_ack_packet))
end_state.update_read_pointer(read_ptr)
end_state.set_update_completed()
# now state is updated, read back values for read pointer and
# last operation performed
last_operation = end_state.last_buffer_operation
read_ptr = end_state.current_read
# now read_ptr is updated, check memory to read
if read_ptr < write_ptr:
length = write_ptr - read_ptr
data = self._transceiver.read_memory(x, y, read_ptr, length)
self._received_data.flushing_data_from_region(
x, y, p, region_to_read, data)
elif read_ptr > write_ptr:
length = end_ptr - read_ptr
data = self._transceiver.read_memory(x, y, read_ptr, length)
self._received_data.store_data_in_region_buffer(
x, y, p, region_to_read, data)
read_ptr = start_ptr
length = write_ptr - read_ptr
data = self._transceiver.read_memory(x, y, read_ptr, length)
self._received_data.flushing_data_from_region(
x, y, p, region_to_read, data)
elif (read_ptr == write_ptr and
last_operation == spinn_front_end_constants.
BUFFERING_OPERATIONS.BUFFER_WRITE.value):
length = end_ptr - read_ptr
data = self._transceiver.read_memory(x, y, read_ptr, length)
self._received_data.store_data_in_region_buffer(
x, y, p, region_to_read, data)
read_ptr = start_ptr
length = write_ptr - read_ptr
data = self._transceiver.read_memory(x, y, read_ptr, length)
self._received_data.flushing_data_from_region(
x, y, p, region_to_read, data)
elif (read_ptr == write_ptr and
last_operation == spinn_front_end_constants.
BUFFERING_OPERATIONS.BUFFER_READ.value):
data = bytearray()
self._received_data.flushing_data_from_region(
x, y, p, region_to_read, data)
# data flush has been completed - return appropriate data
# the two returns can be exchanged - one returns data and the other
# returns a pointer to the structure holding the data
return self._received_data.get_region_data_pointer(
x, y, p, region_to_read)
def get_spikes(
self, txrx, placements, graph_mapper, compatible_output=False):
"""
Return a 2-column numpy array containing cell ids and spike times for
recorded cells. This is read directly from the memory for the board.
:param transceiver:
:param placements:
:param graph_mapper:
:param compatible_output:
"""
logger.info("Getting spikes for {}".format(self._label))
# Find all the sub-vertices that this pynn_population.py exists on
subvertices = graph_mapper.get_subvertices_from_vertex(self)
progress_bar = ProgressBar(len(subvertices), "Getting spikes")
results = list()
for subvertex in subvertices:
placement = placements.get_placement_of_subvertex(subvertex)
(x, y, p) = placement.x, placement.y, placement.p
subvertex_slice = graph_mapper.get_subvertex_slice(subvertex)
lo_atom = subvertex_slice.lo_atom
hi_atom = subvertex_slice.hi_atom
logger.debug("Reading spikes from chip {}, {}, core {}, "
"lo_atom {} hi_atom {}".format(
x, y, p, lo_atom, hi_atom))
# Get the App Data for the core
app_data_base_address = \
txrx.get_cpu_information_from_core(x, y, p).user[0]
# Get the position of the spike buffer
spike_region_base_address_offset = \
dsg_utility_calls.get_region_base_address_offset(
app_data_base_address,
self._SPIKE_SOURCE_REGIONS
.SPIKE_DATA_RECORDED_REGION.value)
spike_region_base_address_buf = buffer(txrx.read_memory(
x, y, spike_region_base_address_offset, 4))
spike_region_base_address = struct.unpack_from(
"<I", spike_region_base_address_buf)[0]
spike_region_base_address += app_data_base_address
# Read the spike data size
number_of_bytes_written_buf = buffer(txrx.read_memory(
x, y, spike_region_base_address, 4))
number_of_bytes_written = struct.unpack_from(
"<I", number_of_bytes_written_buf)[0]
# check that the number of spikes written is smaller or the same as
# the size of the memory region we allocated for spikes
send_buffer = self._get_spike_send_buffer(subvertex_slice)
if number_of_bytes_written > send_buffer.total_region_size:
raise exceptions.MemReadException(
"the amount of memory written ({}) was larger than was "
"allocated for it ({})"
.format(number_of_bytes_written,
send_buffer.total_region_size))
# Read the spikes
logger.debug("Reading {} ({}) bytes starting at {} + 4"
.format(number_of_bytes_written,
hex(number_of_bytes_written),
hex(spike_region_base_address)))
spike_data_block = txrx.read_memory(
x, y, spike_region_base_address + 4, number_of_bytes_written)
# translate block of spikes into EIEIO messages
offset = 0
while offset <= number_of_bytes_written - 4:
eieio_header = EIEIODataHeader.from_bytestring(
spike_data_block, offset)
offset += eieio_header.size
timestamps = numpy.repeat([eieio_header.payload_base],
eieio_header.count)
keys = numpy.frombuffer(
spike_data_block, dtype="<u4", count=eieio_header.count,
offset=offset)
neuron_ids = ((keys - subvertex.base_key) +
subvertex_slice.lo_atom)
offset += eieio_header.count * 4
results.append(numpy.dstack((neuron_ids, timestamps))[0])
# complete the buffer
progress_bar.update()
progress_bar.end()
result = numpy.vstack(results)
result = result[numpy.lexsort((result[:, 1], result[:, 0]))]
return result
def _retrieve_synaptic_block(
self, placements, transceiver, pre_subvertex, pre_n_atoms,
post_subvertex, routing_infos, subgraph):
"""
reads in a synaptic block from a given processor and subvertex on the
machine.
"""
post_placement = placements.get_placement_of_subvertex(post_subvertex)
post_x, post_y, post_p = \
post_placement.x, post_placement.y, post_placement.p
# either read in the master pop table or retrieve it from storage
master_pop_base_mem_address, app_data_base_address = \
self._population_table_type.locate_master_pop_table_base_address(
post_x, post_y, post_p, transceiver,
constants.POPULATION_BASED_REGIONS.POPULATION_TABLE.value)
incoming_edges = subgraph.incoming_subedges_from_subvertex(
post_subvertex)
incoming_key_combo = None
for subedge in incoming_edges:
if subedge.pre_subvertex == pre_subvertex:
routing_info = \
routing_infos.get_subedge_information_from_subedge(subedge)
keys_and_masks = routing_info.keys_and_masks
incoming_key_combo = keys_and_masks[0].key
break
maxed_row_length, synaptic_block_base_address_offset = \
self._population_table_type.extract_synaptic_matrix_data_location(
incoming_key_combo, master_pop_base_mem_address,
transceiver, post_x, post_y)
block = None
if maxed_row_length > 0:
# calculate the synaptic block size in words
synaptic_block_size = (pre_n_atoms * 4 *
(constants.SYNAPTIC_ROW_HEADER_WORDS +
maxed_row_length))
# read in the base address of the synaptic matrix in the app region
# table
synapse_region_base_address_location = \
dsg_utilities.get_region_base_address_offset(
app_data_base_address,
constants.POPULATION_BASED_REGIONS.SYNAPTIC_MATRIX.value)
# read in the memory address of the synaptic_region base address
synapse_region_base_address = helpful_functions.read_data(
post_x, post_y, synapse_region_base_address_location, 4,
"<I", transceiver)
# the base address of the synaptic block in absolute terms is the
# app base, plus the synaptic matrix base plus the offset
synaptic_block_base_address = (app_data_base_address +
synapse_region_base_address +
synaptic_block_base_address_offset)
# read in and return the synaptic block
block = transceiver.read_memory(
post_x, post_y, synaptic_block_base_address,
synaptic_block_size)
if len(block) != synaptic_block_size:
raise exceptions.SynapticBlockReadException(
"Not enough data has been read"
" (aka, something funkky happened)")
return block, maxed_row_length
