def _execute(txrx, machine, app_id, x, y, p, data_spec_path):
# pylint: disable=too-many-arguments, too-many-locals
# build specification reader
reader = FileDataReader(data_spec_path)
# maximum available memory
# however system updates the memory available
# independently, so the check on the space available actually
# happens when memory is allocated
# generate data spec executor
executor = DataSpecificationExecutor(
reader,
machine.get_chip_at(x, y).sdram.size)
# run data spec executor
try:
# bytes_used_by_spec, bytes_written_by_spec = \
executor.execute()
except DataSpecificationException:
logger.error("Error executing data specification for {}, {}, {}",
x, y, p)
raise
bytes_used_by_spec = executor.get_constructed_data_size()
# allocate memory where the app data is going to be written; this
# raises an exception in case there is not enough SDRAM to allocate
start_address = txrx.malloc_sdram(x, y, bytes_used_by_spec, app_id)
# Write the header and pointer table and load it
header = executor.get_header()
pointer_table = executor.get_pointer_table(start_address)
data_to_write = numpy.concatenate((header, pointer_table)).tostring()
txrx.write_memory(x, y, start_address, data_to_write)
bytes_written_by_spec = len(data_to_write)
# Write each region
for region_id in _MEM_REGIONS:
region = executor.get_region(region_id)
if region is not None:
max_pointer = region.max_write_pointer
if not region.unfilled and max_pointer > 0:
# Get the data up to what has been written
data = region.region_data[:max_pointer]
# Write the data to the position
txrx.write_memory(x, y, pointer_table[region_id], data)
bytes_written_by_spec += len(data)
# set user 0 register appropriately to the application data
write_address_to_user0(txrx, x, y, p, start_address)
return {
'start_address': start_address,
'memory_used': bytes_used_by_spec,
'memory_written': bytes_written_by_spec
}
def __execute(self, core, reader, writer_func):
x, y, p = core
# Maximum available memory.
# However, system updates the memory available independently, so the
# space available check actually happens when memory is allocated.
memory_available = self._machine.get_chip_at(x, y).sdram.size
# generate data spec executor
executor = DataSpecificationExecutor(reader, memory_available)
# run data spec executor
try:
executor.execute()
except DataSpecificationException:
logger.error("Error executing data specification for {}, {}, {}",
x, y, p)
raise
bytes_allocated = executor.get_constructed_data_size()
# allocate memory where the app data is going to be written; this
# raises an exception in case there is not enough SDRAM to allocate
start_address = self._txrx.malloc_sdram(x, y, bytes_allocated,
self._app_id)
# Do the actual writing ------------------------------------
# Write the header and pointer table
header = executor.get_header()
pointer_table = executor.get_pointer_table(start_address)
data_to_write = numpy.concatenate((header, pointer_table)).tostring()
# NB: DSE meta-block is always small (i.e., one SDP write)
self._txrx.write_memory(x, y, start_address, data_to_write)
bytes_written = len(data_to_write)
# Write each region
for region_id in _MEM_REGIONS:
region = executor.get_region(region_id)
if region is None:
continue
max_pointer = region.max_write_pointer
if region.unfilled or max_pointer == 0:
continue
# Get the data up to what has been written
data = region.region_data[:max_pointer]
# Write the data to the position
writer_func(x, y, pointer_table[region_id], data)
bytes_written += len(data)
# set user 0 register appropriately to the application data
write_address_to_user0(self._txrx, x, y, p, start_address)
return DataWritten(start_address, bytes_allocated, bytes_written)
def __python_execute(self, core, reader, writer_func, base_address,
size_allocated):
"""
:param tuple(int,int,int) core:
:param ~.AbstractDataReader reader:
:param callable(tuple(int,int,int,bytearray),None) writer_func:
:param int base_address:
:param int size_allocated:
:rtype: DataWritten
"""
x, y, p = core
# Maximum available memory.
# However, system updates the memory available independently, so the
# space available check actually happens when memory is allocated.
memory_available = self._machine.get_chip_at(x, y).sdram.size
# generate data spec executor
executor = DataSpecificationExecutor(reader, memory_available)
# run data spec executor
try:
executor.execute()
except DataSpecificationException:
logger.error("Error executing data specification for {}, {}, {}",
x, y, p)
raise
# Do the actual writing ------------------------------------
# Write the header and pointer table
header = executor.get_header()
pointer_table = executor.get_pointer_table(base_address)
data_to_write = numpy.concatenate((header, pointer_table)).tostring()
# NB: DSE meta-block is always small (i.e., one SDP write)
self._txrx.write_memory(x, y, base_address, data_to_write)
bytes_written = len(data_to_write)
# Write each region
for region_id in _MEM_REGIONS:
region = executor.get_region(region_id)
if region is None:
continue
max_pointer = region.max_write_pointer
if region.unfilled or max_pointer == 0:
continue
# Get the data up to what has been written
data = region.region_data[:max_pointer]
# Write the data to the position
writer_func(x, y, pointer_table[region_id], data)
bytes_written += len(data)
return DataWritten(base_address, size_allocated, bytes_written)
def __call__(self, transceiver, machine, app_id, dsg_targets):
"""
:param machine: the python representation of the spinnaker machine
:param transceiver: the spinnman instance
:param app_id: the application ID of the simulation
:param dsg_targets: map of placement to file path
:return: map of placement and dsg data, and loaded data flag.
"""
processor_to_app_data_base_address = dict()
# create a progress bar for end users
progress = ProgressBar(
dsg_targets, "Executing data specifications and loading data")
for ((x, y, p),
data_spec_file_path) in progress.over(dsg_targets.iteritems()):
# build specification reader
data_spec_file_path = dsg_targets[x, y, p]
data_spec_reader = FileDataReader(data_spec_file_path)
# maximum available memory
# however system updates the memory available
# independently, so the check on the space available actually
# happens when memory is allocated
chip = machine.get_chip_at(x, y)
memory_available = chip.sdram.size
# generate data spec executor
executor = DataSpecificationExecutor(data_spec_reader,
memory_available)
# run data spec executor
try:
# bytes_used_by_spec, bytes_written_by_spec = \
executor.execute()
except DataSpecificationException as e:
logger.error(
"Error executing data specification for {}, {}, {}".format(
x, y, p))
raise e
bytes_used_by_spec = executor.get_constructed_data_size()
# allocate memory where the app data is going to be written
# this raises an exception in case there is not enough
# SDRAM to allocate
start_address = transceiver.malloc_sdram(x, y, bytes_used_by_spec,
app_id)
# Write the header and pointer table and load it
header = executor.get_header()
pointer_table = executor.get_pointer_table(start_address)
data_to_write = numpy.concatenate(
(header, pointer_table)).tostring()
transceiver.write_memory(x, y, start_address, data_to_write)
bytes_written_by_spec = len(data_to_write)
# Write each region
for region_id in range(constants.MAX_MEM_REGIONS):
region = executor.get_region(region_id)
if region is not None:
max_pointer = region.max_write_pointer
if not region.unfilled and max_pointer > 0:
# Get the data up to what has been written
data = region.region_data[:max_pointer]
# Write the data to the position
position = pointer_table[region_id]
transceiver.write_memory(x, y, position, data)
bytes_written_by_spec += len(data)
# set user 0 register appropriately to the application data
user_0_address = \
transceiver.get_user_0_register_address_from_core(x, y, p)
start_address_encoded = \
buffer(struct.pack("<I", start_address))
transceiver.write_memory(x, y, user_0_address,
start_address_encoded)
# write information for the memory map report
processor_to_app_data_base_address[x, y, p] = {
'start_address': start_address,
'memory_used': bytes_used_by_spec,
'memory_written': bytes_written_by_spec
}
return processor_to_app_data_base_address, True
def test_simple_spec(self):
# Create a sdram just to set max chip size
SDRAM(1000)
# Write a data spec to execute
temp_spec = mktemp()
spec_writer = io.FileIO(temp_spec, "w")
spec = DataSpecificationGenerator(spec_writer)
spec.reserve_memory_region(0, 100)
spec.reserve_memory_region(1, 200, empty=True)
spec.reserve_memory_region(2, 4)
spec.reserve_memory_region(3, 12, reference=1)
spec.reference_memory_region(4, 2)
spec.switch_write_focus(0)
spec.write_array([0, 1, 2])
spec.set_write_pointer(20)
spec.write_value(4)
spec.switch_write_focus(2)
spec.write_value(3)
spec.set_write_pointer(0)
spec.write_value(10)
spec.end_specification()
# Execute the spec
spec_reader = io.FileIO(temp_spec, "r")
executor = DataSpecificationExecutor(spec_reader, 400)
executor.execute()
# Test the size
header_and_table_size = (constants.MAX_MEM_REGIONS + 2) * 4
self.assertEqual(executor.get_constructed_data_size(),
header_and_table_size + 100 + 200 + 4 + 12)
# Test the unused regions
for region in range(5, constants.MAX_MEM_REGIONS):
self.assertIsNone(executor.get_region(region))
# Test region 0
region_0 = executor.get_region(0)
self.assertEqual(region_0.allocated_size, 100)
self.assertEqual(region_0.max_write_pointer, 24)
self.assertFalse(region_0.unfilled)
self.assertIsNone(region_0.reference)
self.assertEqual(region_0.region_data[:region_0.max_write_pointer],
struct.pack("<IIIIII", 0, 1, 2, 0, 0, 4))
# Test region 1
region_1 = executor.get_region(1)
self.assertIsInstance(region_1, MemoryRegionReal)
self.assertEqual(region_1.allocated_size, 200)
self.assertTrue(region_1.unfilled)
self.assertIsNone(region_1.reference)
# Test region 2
region_2 = executor.get_region(2)
self.assertIsInstance(region_2, MemoryRegionReal)
self.assertEqual(region_2.allocated_size, 4)
self.assertIsNone(region_2.reference)
self.assertEqual(region_2.region_data, struct.pack("<I", 10))
# Test region 3
region_3 = executor.get_region(3)
self.assertIsInstance(region_3, MemoryRegionReal)
self.assertEqual(region_3.allocated_size, 12)
self.assertEqual(region_3.reference, 1)
self.assertEqual(executor.referenceable_regions, [3])
# Test region 4
region_4 = executor.get_region(4)
self.assertIsInstance(region_4, MemoryRegionReference)
self.assertEqual(region_4.ref, 2)
self.assertEqual(executor.references_to_fill, [4])
# Test the pointer table
table = executor.get_pointer_table(0)
self.assertEqual(len(table), constants.MAX_MEM_REGIONS)
self.assertEqual(table[0], header_and_table_size)
self.assertEqual(table[1], header_and_table_size + 100)
self.assertEqual(table[2], header_and_table_size + 300)
self.assertEqual(table[3], header_and_table_size + 304)
# 4 is also 0 because it is a reference
for region in range(4, constants.MAX_MEM_REGIONS):
self.assertEqual(table[region], 0)
# Test the header
header = executor.get_header()
self.assertEqual(len(header), 2)
self.assertEqual(header[0], constants.APPDATA_MAGIC_NUM)
self.assertEqual(header[1], constants.DSE_VERSION)
def test_write_data_spec():
unittest_setup()
# UGLY but the mock transceiver NEED generate_on_machine to be False
AbstractGenerateConnectorOnMachine.generate_on_machine = say_false
machine = virtual_machine(2, 2)
p.setup(1.0)
load_config()
p.set_number_of_neurons_per_core(p.IF_curr_exp, 100)
pre_pop = p.Population(10,
p.IF_curr_exp(),
label="Pre",
additional_parameters={
"splitter":
SplitterAbstractPopulationVertexSlice()
})
post_pop = p.Population(10,
p.IF_curr_exp(),
label="Post",
additional_parameters={
"splitter":
SplitterAbstractPopulationVertexSlice()
})
proj_one_to_one_1 = p.Projection(pre_pop, post_pop, p.OneToOneConnector(),
p.StaticSynapse(weight=1.5, delay=1.0))
proj_one_to_one_2 = p.Projection(pre_pop, post_pop, p.OneToOneConnector(),
p.StaticSynapse(weight=2.5, delay=2.0))
proj_all_to_all = p.Projection(
pre_pop, post_pop, p.AllToAllConnector(allow_self_connections=False),
p.StaticSynapse(weight=4.5, delay=4.0))
from_list_list = [(i, i, i, (i * 5) + 1) for i in range(10)]
proj_from_list = p.Projection(pre_pop, post_pop,
p.FromListConnector(from_list_list),
p.StaticSynapse())
app_graph = globals_variables.get_simulator().original_application_graph
context = {"ApplicationGraph": app_graph}
with (injection_context(context)):
delay_support_adder(app_graph)
machine_graph, _ = spynnaker_splitter_partitioner(
app_graph, machine, 100)
allocator = ZonedRoutingInfoAllocator()
n_keys_map = edge_to_n_keys_mapper(machine_graph)
routing_info = allocator.__call__(machine_graph,
n_keys_map,
flexible=False)
post_vertex = next(iter(post_pop._vertex.machine_vertices))
post_vertex_slice = post_vertex.vertex_slice
post_vertex_placement = Placement(post_vertex, 0, 0, 3)
temp_spec = tempfile.mktemp()
spec = DataSpecificationGenerator(io.FileIO(temp_spec, "wb"), None)
synaptic_matrices = SynapticMatrices(post_vertex_slice,
n_synapse_types=2,
all_single_syn_sz=10000,
synaptic_matrix_region=1,
direct_matrix_region=2,
poptable_region=3,
connection_builder_region=4)
synaptic_matrices.write_synaptic_data(
spec,
post_pop._vertex.incoming_projections,
all_syn_block_sz=10000,
weight_scales=[32, 32],
routing_info=routing_info)
spec.end_specification()
with io.FileIO(temp_spec, "rb") as spec_reader:
executor = DataSpecificationExecutor(spec_reader, 20000)
executor.execute()
all_data = bytearray()
all_data.extend(bytearray(executor.get_header()))
all_data.extend(bytearray(executor.get_pointer_table(0)))
for r in range(MAX_MEM_REGIONS):
region = executor.get_region(r)
if region is not None:
all_data.extend(region.region_data)
transceiver = MockTransceiverRawData(all_data)
report_folder = mkdtemp()
try:
connections_1 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_one_to_one_1._projection_edge,
proj_one_to_one_1._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_1) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 1.5 for conn in connections_1])
assert all([conn["delay"] == 1.0 for conn in connections_1])
connections_2 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_one_to_one_2._projection_edge,
proj_one_to_one_2._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_2) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 2.5 for conn in connections_2])
assert all([conn["delay"] == 2.0 for conn in connections_2])
connections_3 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_all_to_all._projection_edge,
proj_all_to_all._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_3) == 100
assert all([conn["weight"] == 4.5 for conn in connections_3])
assert all([conn["delay"] == 4.0 for conn in connections_3])
connections_4 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_from_list._projection_edge,
proj_from_list._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_4) == len(from_list_list)
list_weights = [values[2] for values in from_list_list]
list_delays = [values[3] for values in from_list_list]
assert all(list_weights == connections_4["weight"])
assert all(list_delays == connections_4["delay"])
finally:
shutil.rmtree(report_folder, ignore_errors=True)
def test_write_data_spec():
unittest_setup()
# UGLY but the mock transceiver NEED generate_on_machine to be False
AbstractGenerateConnectorOnMachine.generate_on_machine = say_false
machine = virtual_machine(2, 2)
p.setup(1.0)
load_config()
p.set_number_of_neurons_per_core(p.IF_curr_exp, 100)
pre_pop = p.Population(
10, p.IF_curr_exp(), label="Pre",
additional_parameters={
"splitter": SplitterAbstractPopulationVertexSlice()})
post_pop = p.Population(
10, p.IF_curr_exp(), label="Post",
additional_parameters={
"splitter": SplitterAbstractPopulationVertexSlice()})
proj_one_to_one_1 = p.Projection(
pre_pop, post_pop, p.OneToOneConnector(),
p.StaticSynapse(weight=1.5, delay=1.0))
proj_one_to_one_2 = p.Projection(
pre_pop, post_pop, p.OneToOneConnector(),
p.StaticSynapse(weight=2.5, delay=2.0))
proj_all_to_all = p.Projection(
pre_pop, post_pop, p.AllToAllConnector(allow_self_connections=False),
p.StaticSynapse(weight=4.5, delay=4.0))
# spynnaker8.setup(timestep=1)
# # Add an sdram so max SDRAM is high enough
# SDRAM(10000)
#
# set_config("Simulation", "one_to_one_connection_dtcm_max_bytes", 40)
#
# placements = Placements()
# pre_app_population = MockPopulation(10, "mock pop pre")
# pre_app_vertex = SimpleTestVertex(10, label="pre")
# pre_app_vertex.splitter = MockSplitter()
# pre_app_vertex.splitter._called = True
# pre_vertex_slice = Slice(0, 9)
#
# post_app_population = MockPopulation(10, "mock pop post")
# pre_vertex = pre_app_vertex.create_machine_vertex(
# pre_vertex_slice, None)
# placements.add_placement(Placement(pre_vertex, 0, 0, 1))
# post_app_vertex = SimpleTestVertex(10, label="post")
# post_app_vertex.splitter = MockSplitter()
# post_app_vertex.splitter._called = True
# post_vertex_slice = Slice(0, 9)
# post_vertex = post_app_vertex.create_machine_vertex(
# post_vertex_slice, None)
# post_vertex_placement = Placement(post_vertex, 0, 0, 2)
# placements.add_placement(post_vertex_placement)
# delay_app_vertex = DelayExtensionVertex(
# 10, 16, 51, pre_app_vertex, label="delay")
# delay_app_vertex.set_new_n_delay_stages_and_delay_per_stage(
# 16, 51)
# delay_app_vertex.splitter = SplitterDelayVertexSlice(
# pre_app_vertex.splitter)
# delay_vertex = DelayExtensionMachineVertex(
# resources_required=None, label="", constraints=[],
# app_vertex=delay_app_vertex, vertex_slice=post_vertex_slice)
# placements.add_placement(Placement(delay_vertex, 0, 0, 3))
# one_to_one_connector_1 = OneToOneConnector(None)
# direct_synapse_information_1 = SynapseInformation(
# one_to_one_connector_1, pre_app_population, post_app_population,
# False, False, None, SynapseDynamicsStatic(), 0, True, 1.5, 1.0)
# one_to_one_connector_1.set_projection_information(
# direct_synapse_information_1)
# one_to_one_connector_2 = OneToOneConnector(None)
# direct_synapse_information_2 = SynapseInformation(
# one_to_one_connector_2, pre_app_population, post_app_population,
# False, False, None, SynapseDynamicsStatic(), 1, True, 2.5, 2.0)
# one_to_one_connector_2.set_projection_information(
# direct_synapse_information_2)
# all_to_all_connector = AllToAllConnector(False)
# all_to_all_synapse_information = SynapseInformation(
# all_to_all_connector, pre_app_population, post_app_population,
# False, False, None, SynapseDynamicsStatic(), 0, True, 4.5, 4.0)
# all_to_all_connector.set_projection_information(
# all_to_all_synapse_information)
from_list_list = [(i, i, i, (i * 5) + 1) for i in range(10)]
proj_from_list = p.Projection(
pre_pop, post_pop, p.FromListConnector(from_list_list),
p.StaticSynapse())
app_graph = globals_variables.get_simulator().original_application_graph
context = {
"ApplicationGraph": app_graph
}
with (injection_context(context)):
delay_adder = DelaySupportAdder()
delay_adder.__call__(app_graph)
partitioner = SpynnakerSplitterPartitioner()
machine_graph, _ = partitioner.__call__(app_graph, machine, 100)
allocator = ZonedRoutingInfoAllocator()
n_keys_mapper = EdgeToNKeysMapper()
n_keys_map = n_keys_mapper.__call__(machine_graph)
routing_info = allocator.__call__(
machine_graph, n_keys_map, flexible=False)
post_vertex = next(iter(post_pop._vertex.machine_vertices))
post_vertex_slice = post_vertex.vertex_slice
post_vertex_placement = Placement(post_vertex, 0, 0, 3)
temp_spec = tempfile.mktemp()
spec = DataSpecificationGenerator(io.FileIO(temp_spec, "wb"), None)
synaptic_matrices = SynapticMatrices(
post_vertex_slice, n_synapse_types=2, all_single_syn_sz=10000,
synaptic_matrix_region=1, direct_matrix_region=2, poptable_region=3,
connection_builder_region=4)
synaptic_matrices.write_synaptic_data(
spec, post_pop._vertex.incoming_projections, all_syn_block_sz=10000,
weight_scales=[32, 32], routing_info=routing_info)
spec.end_specification()
with io.FileIO(temp_spec, "rb") as spec_reader:
executor = DataSpecificationExecutor(spec_reader, 20000)
executor.execute()
all_data = bytearray()
all_data.extend(bytearray(executor.get_header()))
all_data.extend(bytearray(executor.get_pointer_table(0)))
for r in range(MAX_MEM_REGIONS):
region = executor.get_region(r)
if region is not None:
all_data.extend(region.region_data)
transceiver = MockTransceiverRawData(all_data)
report_folder = mkdtemp()
try:
connections_1 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_one_to_one_1._projection_edge,
proj_one_to_one_1._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_1) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 1.5 for conn in connections_1])
assert all([conn["delay"] == 1.0 for conn in connections_1])
connections_2 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_one_to_one_2._projection_edge,
proj_one_to_one_2._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_2) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 2.5 for conn in connections_2])
assert all([conn["delay"] == 2.0 for conn in connections_2])
connections_3 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_all_to_all._projection_edge,
proj_all_to_all._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_3) == 100
assert all([conn["weight"] == 4.5 for conn in connections_3])
assert all([conn["delay"] == 4.0 for conn in connections_3])
connections_4 = numpy.concatenate(
synaptic_matrices.get_connections_from_machine(
transceiver, post_vertex_placement,
proj_from_list._projection_edge,
proj_from_list._synapse_information))
# Check that all the connections have the right weight and delay
assert len(connections_4) == len(from_list_list)
list_weights = [values[2] for values in from_list_list]
list_delays = [values[3] for values in from_list_list]
assert all(list_weights == connections_4["weight"])
assert all(list_delays == connections_4["delay"])
finally:
shutil.rmtree(report_folder, ignore_errors=True)