def _execute(txrx, machine, app_id, x, y, p, data_spec_path): # pylint: disable=too-many-arguments, too-many-locals # build specification reader reader = FileDataReader(data_spec_path) # maximum available memory # however system updates the memory available # independently, so the check on the space available actually # happens when memory is allocated # generate data spec executor executor = DataSpecificationExecutor( reader, machine.get_chip_at(x, y).sdram.size) # run data spec executor try: # bytes_used_by_spec, bytes_written_by_spec = \ executor.execute() except DataSpecificationException: logger.error("Error executing data specification for {}, {}, {}", x, y, p) raise bytes_used_by_spec = executor.get_constructed_data_size() # allocate memory where the app data is going to be written; this # raises an exception in case there is not enough SDRAM to allocate start_address = txrx.malloc_sdram(x, y, bytes_used_by_spec, app_id) # Write the header and pointer table and load it header = executor.get_header() pointer_table = executor.get_pointer_table(start_address) data_to_write = numpy.concatenate((header, pointer_table)).tostring() txrx.write_memory(x, y, start_address, data_to_write) bytes_written_by_spec = len(data_to_write) # Write each region for region_id in _MEM_REGIONS: region = executor.get_region(region_id) if region is not None: max_pointer = region.max_write_pointer if not region.unfilled and max_pointer > 0: # Get the data up to what has been written data = region.region_data[:max_pointer] # Write the data to the position txrx.write_memory(x, y, pointer_table[region_id], data) bytes_written_by_spec += len(data) # set user 0 register appropriately to the application data write_address_to_user0(txrx, x, y, p, start_address) return { 'start_address': start_address, 'memory_used': bytes_used_by_spec, 'memory_written': bytes_written_by_spec }
def __execute(self, core, reader, writer_func): x, y, p = core # Maximum available memory. # However, system updates the memory available independently, so the # space available check actually happens when memory is allocated. memory_available = self._machine.get_chip_at(x, y).sdram.size # generate data spec executor executor = DataSpecificationExecutor(reader, memory_available) # run data spec executor try: executor.execute() except DataSpecificationException: logger.error("Error executing data specification for {}, {}, {}", x, y, p) raise bytes_allocated = executor.get_constructed_data_size() # allocate memory where the app data is going to be written; this # raises an exception in case there is not enough SDRAM to allocate start_address = self._txrx.malloc_sdram(x, y, bytes_allocated, self._app_id) # Do the actual writing ------------------------------------ # Write the header and pointer table header = executor.get_header() pointer_table = executor.get_pointer_table(start_address) data_to_write = numpy.concatenate((header, pointer_table)).tostring() # NB: DSE meta-block is always small (i.e., one SDP write) self._txrx.write_memory(x, y, start_address, data_to_write) bytes_written = len(data_to_write) # Write each region for region_id in _MEM_REGIONS: region = executor.get_region(region_id) if region is None: continue max_pointer = region.max_write_pointer if region.unfilled or max_pointer == 0: continue # Get the data up to what has been written data = region.region_data[:max_pointer] # Write the data to the position writer_func(x, y, pointer_table[region_id], data) bytes_written += len(data) # set user 0 register appropriately to the application data write_address_to_user0(self._txrx, x, y, p, start_address) return DataWritten(start_address, bytes_allocated, bytes_written)
def __python_execute(self, core, reader, writer_func, base_address, size_allocated): """ :param tuple(int,int,int) core: :param ~.AbstractDataReader reader: :param callable(tuple(int,int,int,bytearray),None) writer_func: :param int base_address: :param int size_allocated: :rtype: DataWritten """ x, y, p = core # Maximum available memory. # However, system updates the memory available independently, so the # space available check actually happens when memory is allocated. memory_available = self._machine.get_chip_at(x, y).sdram.size # generate data spec executor executor = DataSpecificationExecutor(reader, memory_available) # run data spec executor try: executor.execute() except DataSpecificationException: logger.error("Error executing data specification for {}, {}, {}", x, y, p) raise # Do the actual writing ------------------------------------ # Write the header and pointer table header = executor.get_header() pointer_table = executor.get_pointer_table(base_address) data_to_write = numpy.concatenate((header, pointer_table)).tostring() # NB: DSE meta-block is always small (i.e., one SDP write) self._txrx.write_memory(x, y, base_address, data_to_write) bytes_written = len(data_to_write) # Write each region for region_id in _MEM_REGIONS: region = executor.get_region(region_id) if region is None: continue max_pointer = region.max_write_pointer if region.unfilled or max_pointer == 0: continue # Get the data up to what has been written data = region.region_data[:max_pointer] # Write the data to the position writer_func(x, y, pointer_table[region_id], data) bytes_written += len(data) return DataWritten(base_address, size_allocated, bytes_written)
def __call__(self, transceiver, machine, app_id, dsg_targets): """ :param machine: the python representation of the spinnaker machine :param transceiver: the spinnman instance :param app_id: the application ID of the simulation :param dsg_targets: map of placement to file path :return: map of placement and dsg data, and loaded data flag. """ processor_to_app_data_base_address = dict() # create a progress bar for end users progress = ProgressBar( dsg_targets, "Executing data specifications and loading data") for ((x, y, p), data_spec_file_path) in progress.over(dsg_targets.iteritems()): # build specification reader data_spec_file_path = dsg_targets[x, y, p] data_spec_reader = FileDataReader(data_spec_file_path) # maximum available memory # however system updates the memory available # independently, so the check on the space available actually # happens when memory is allocated chip = machine.get_chip_at(x, y) memory_available = chip.sdram.size # generate data spec executor executor = DataSpecificationExecutor(data_spec_reader, memory_available) # run data spec executor try: # bytes_used_by_spec, bytes_written_by_spec = \ executor.execute() except DataSpecificationException as e: logger.error( "Error executing data specification for {}, {}, {}".format( x, y, p)) raise e bytes_used_by_spec = executor.get_constructed_data_size() # allocate memory where the app data is going to be written # this raises an exception in case there is not enough # SDRAM to allocate start_address = transceiver.malloc_sdram(x, y, bytes_used_by_spec, app_id) # Write the header and pointer table and load it header = executor.get_header() pointer_table = executor.get_pointer_table(start_address) data_to_write = numpy.concatenate( (header, pointer_table)).tostring() transceiver.write_memory(x, y, start_address, data_to_write) bytes_written_by_spec = len(data_to_write) # Write each region for region_id in range(constants.MAX_MEM_REGIONS): region = executor.get_region(region_id) if region is not None: max_pointer = region.max_write_pointer if not region.unfilled and max_pointer > 0: # Get the data up to what has been written data = region.region_data[:max_pointer] # Write the data to the position position = pointer_table[region_id] transceiver.write_memory(x, y, position, data) bytes_written_by_spec += len(data) # set user 0 register appropriately to the application data user_0_address = \ transceiver.get_user_0_register_address_from_core(x, y, p) start_address_encoded = \ buffer(struct.pack("<I", start_address)) transceiver.write_memory(x, y, user_0_address, start_address_encoded) # write information for the memory map report processor_to_app_data_base_address[x, y, p] = { 'start_address': start_address, 'memory_used': bytes_used_by_spec, 'memory_written': bytes_written_by_spec } return processor_to_app_data_base_address, True
def test_simple_spec(self): # Create a sdram just to set max chip size SDRAM(1000) # Write a data spec to execute temp_spec = mktemp() spec_writer = io.FileIO(temp_spec, "w") spec = DataSpecificationGenerator(spec_writer) spec.reserve_memory_region(0, 100) spec.reserve_memory_region(1, 200, empty=True) spec.reserve_memory_region(2, 4) spec.reserve_memory_region(3, 12, reference=1) spec.reference_memory_region(4, 2) spec.switch_write_focus(0) spec.write_array([0, 1, 2]) spec.set_write_pointer(20) spec.write_value(4) spec.switch_write_focus(2) spec.write_value(3) spec.set_write_pointer(0) spec.write_value(10) spec.end_specification() # Execute the spec spec_reader = io.FileIO(temp_spec, "r") executor = DataSpecificationExecutor(spec_reader, 400) executor.execute() # Test the size header_and_table_size = (constants.MAX_MEM_REGIONS + 2) * 4 self.assertEqual(executor.get_constructed_data_size(), header_and_table_size + 100 + 200 + 4 + 12) # Test the unused regions for region in range(5, constants.MAX_MEM_REGIONS): self.assertIsNone(executor.get_region(region)) # Test region 0 region_0 = executor.get_region(0) self.assertEqual(region_0.allocated_size, 100) self.assertEqual(region_0.max_write_pointer, 24) self.assertFalse(region_0.unfilled) self.assertIsNone(region_0.reference) self.assertEqual(region_0.region_data[:region_0.max_write_pointer], struct.pack("<IIIIII", 0, 1, 2, 0, 0, 4)) # Test region 1 region_1 = executor.get_region(1) self.assertIsInstance(region_1, MemoryRegionReal) self.assertEqual(region_1.allocated_size, 200) self.assertTrue(region_1.unfilled) self.assertIsNone(region_1.reference) # Test region 2 region_2 = executor.get_region(2) self.assertIsInstance(region_2, MemoryRegionReal) self.assertEqual(region_2.allocated_size, 4) self.assertIsNone(region_2.reference) self.assertEqual(region_2.region_data, struct.pack("<I", 10)) # Test region 3 region_3 = executor.get_region(3) self.assertIsInstance(region_3, MemoryRegionReal) self.assertEqual(region_3.allocated_size, 12) self.assertEqual(region_3.reference, 1) self.assertEqual(executor.referenceable_regions, [3]) # Test region 4 region_4 = executor.get_region(4) self.assertIsInstance(region_4, MemoryRegionReference) self.assertEqual(region_4.ref, 2) self.assertEqual(executor.references_to_fill, [4]) # Test the pointer table table = executor.get_pointer_table(0) self.assertEqual(len(table), constants.MAX_MEM_REGIONS) self.assertEqual(table[0], header_and_table_size) self.assertEqual(table[1], header_and_table_size + 100) self.assertEqual(table[2], header_and_table_size + 300) self.assertEqual(table[3], header_and_table_size + 304) # 4 is also 0 because it is a reference for region in range(4, constants.MAX_MEM_REGIONS): self.assertEqual(table[region], 0) # Test the header header = executor.get_header() self.assertEqual(len(header), 2) self.assertEqual(header[0], constants.APPDATA_MAGIC_NUM) self.assertEqual(header[1], constants.DSE_VERSION)
def test_write_data_spec(): unittest_setup() # UGLY but the mock transceiver NEED generate_on_machine to be False AbstractGenerateConnectorOnMachine.generate_on_machine = say_false machine = virtual_machine(2, 2) p.setup(1.0) load_config() p.set_number_of_neurons_per_core(p.IF_curr_exp, 100) pre_pop = p.Population(10, p.IF_curr_exp(), label="Pre", additional_parameters={ "splitter": SplitterAbstractPopulationVertexSlice() }) post_pop = p.Population(10, p.IF_curr_exp(), label="Post", additional_parameters={ "splitter": SplitterAbstractPopulationVertexSlice() }) proj_one_to_one_1 = p.Projection(pre_pop, post_pop, p.OneToOneConnector(), p.StaticSynapse(weight=1.5, delay=1.0)) proj_one_to_one_2 = p.Projection(pre_pop, post_pop, p.OneToOneConnector(), p.StaticSynapse(weight=2.5, delay=2.0)) proj_all_to_all = p.Projection( pre_pop, post_pop, p.AllToAllConnector(allow_self_connections=False), p.StaticSynapse(weight=4.5, delay=4.0)) from_list_list = [(i, i, i, (i * 5) + 1) for i in range(10)] proj_from_list = p.Projection(pre_pop, post_pop, p.FromListConnector(from_list_list), p.StaticSynapse()) app_graph = globals_variables.get_simulator().original_application_graph context = {"ApplicationGraph": app_graph} with (injection_context(context)): delay_support_adder(app_graph) machine_graph, _ = spynnaker_splitter_partitioner( app_graph, machine, 100) allocator = ZonedRoutingInfoAllocator() n_keys_map = edge_to_n_keys_mapper(machine_graph) routing_info = allocator.__call__(machine_graph, n_keys_map, flexible=False) post_vertex = next(iter(post_pop._vertex.machine_vertices)) post_vertex_slice = post_vertex.vertex_slice post_vertex_placement = Placement(post_vertex, 0, 0, 3) temp_spec = tempfile.mktemp() spec = DataSpecificationGenerator(io.FileIO(temp_spec, "wb"), None) synaptic_matrices = SynapticMatrices(post_vertex_slice, n_synapse_types=2, all_single_syn_sz=10000, synaptic_matrix_region=1, direct_matrix_region=2, poptable_region=3, connection_builder_region=4) synaptic_matrices.write_synaptic_data( spec, post_pop._vertex.incoming_projections, all_syn_block_sz=10000, weight_scales=[32, 32], routing_info=routing_info) spec.end_specification() with io.FileIO(temp_spec, "rb") as spec_reader: executor = DataSpecificationExecutor(spec_reader, 20000) executor.execute() all_data = bytearray() all_data.extend(bytearray(executor.get_header())) all_data.extend(bytearray(executor.get_pointer_table(0))) for r in range(MAX_MEM_REGIONS): region = executor.get_region(r) if region is not None: all_data.extend(region.region_data) transceiver = MockTransceiverRawData(all_data) report_folder = mkdtemp() try: connections_1 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_one_to_one_1._projection_edge, proj_one_to_one_1._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_1) == post_vertex_slice.n_atoms assert all([conn["weight"] == 1.5 for conn in connections_1]) assert all([conn["delay"] == 1.0 for conn in connections_1]) connections_2 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_one_to_one_2._projection_edge, proj_one_to_one_2._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_2) == post_vertex_slice.n_atoms assert all([conn["weight"] == 2.5 for conn in connections_2]) assert all([conn["delay"] == 2.0 for conn in connections_2]) connections_3 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_all_to_all._projection_edge, proj_all_to_all._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_3) == 100 assert all([conn["weight"] == 4.5 for conn in connections_3]) assert all([conn["delay"] == 4.0 for conn in connections_3]) connections_4 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_from_list._projection_edge, proj_from_list._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_4) == len(from_list_list) list_weights = [values[2] for values in from_list_list] list_delays = [values[3] for values in from_list_list] assert all(list_weights == connections_4["weight"]) assert all(list_delays == connections_4["delay"]) finally: shutil.rmtree(report_folder, ignore_errors=True)
def test_write_data_spec(): unittest_setup() # UGLY but the mock transceiver NEED generate_on_machine to be False AbstractGenerateConnectorOnMachine.generate_on_machine = say_false machine = virtual_machine(2, 2) p.setup(1.0) load_config() p.set_number_of_neurons_per_core(p.IF_curr_exp, 100) pre_pop = p.Population( 10, p.IF_curr_exp(), label="Pre", additional_parameters={ "splitter": SplitterAbstractPopulationVertexSlice()}) post_pop = p.Population( 10, p.IF_curr_exp(), label="Post", additional_parameters={ "splitter": SplitterAbstractPopulationVertexSlice()}) proj_one_to_one_1 = p.Projection( pre_pop, post_pop, p.OneToOneConnector(), p.StaticSynapse(weight=1.5, delay=1.0)) proj_one_to_one_2 = p.Projection( pre_pop, post_pop, p.OneToOneConnector(), p.StaticSynapse(weight=2.5, delay=2.0)) proj_all_to_all = p.Projection( pre_pop, post_pop, p.AllToAllConnector(allow_self_connections=False), p.StaticSynapse(weight=4.5, delay=4.0)) # spynnaker8.setup(timestep=1) # # Add an sdram so max SDRAM is high enough # SDRAM(10000) # # set_config("Simulation", "one_to_one_connection_dtcm_max_bytes", 40) # # placements = Placements() # pre_app_population = MockPopulation(10, "mock pop pre") # pre_app_vertex = SimpleTestVertex(10, label="pre") # pre_app_vertex.splitter = MockSplitter() # pre_app_vertex.splitter._called = True # pre_vertex_slice = Slice(0, 9) # # post_app_population = MockPopulation(10, "mock pop post") # pre_vertex = pre_app_vertex.create_machine_vertex( # pre_vertex_slice, None) # placements.add_placement(Placement(pre_vertex, 0, 0, 1)) # post_app_vertex = SimpleTestVertex(10, label="post") # post_app_vertex.splitter = MockSplitter() # post_app_vertex.splitter._called = True # post_vertex_slice = Slice(0, 9) # post_vertex = post_app_vertex.create_machine_vertex( # post_vertex_slice, None) # post_vertex_placement = Placement(post_vertex, 0, 0, 2) # placements.add_placement(post_vertex_placement) # delay_app_vertex = DelayExtensionVertex( # 10, 16, 51, pre_app_vertex, label="delay") # delay_app_vertex.set_new_n_delay_stages_and_delay_per_stage( # 16, 51) # delay_app_vertex.splitter = SplitterDelayVertexSlice( # pre_app_vertex.splitter) # delay_vertex = DelayExtensionMachineVertex( # resources_required=None, label="", constraints=[], # app_vertex=delay_app_vertex, vertex_slice=post_vertex_slice) # placements.add_placement(Placement(delay_vertex, 0, 0, 3)) # one_to_one_connector_1 = OneToOneConnector(None) # direct_synapse_information_1 = SynapseInformation( # one_to_one_connector_1, pre_app_population, post_app_population, # False, False, None, SynapseDynamicsStatic(), 0, True, 1.5, 1.0) # one_to_one_connector_1.set_projection_information( # direct_synapse_information_1) # one_to_one_connector_2 = OneToOneConnector(None) # direct_synapse_information_2 = SynapseInformation( # one_to_one_connector_2, pre_app_population, post_app_population, # False, False, None, SynapseDynamicsStatic(), 1, True, 2.5, 2.0) # one_to_one_connector_2.set_projection_information( # direct_synapse_information_2) # all_to_all_connector = AllToAllConnector(False) # all_to_all_synapse_information = SynapseInformation( # all_to_all_connector, pre_app_population, post_app_population, # False, False, None, SynapseDynamicsStatic(), 0, True, 4.5, 4.0) # all_to_all_connector.set_projection_information( # all_to_all_synapse_information) from_list_list = [(i, i, i, (i * 5) + 1) for i in range(10)] proj_from_list = p.Projection( pre_pop, post_pop, p.FromListConnector(from_list_list), p.StaticSynapse()) app_graph = globals_variables.get_simulator().original_application_graph context = { "ApplicationGraph": app_graph } with (injection_context(context)): delay_adder = DelaySupportAdder() delay_adder.__call__(app_graph) partitioner = SpynnakerSplitterPartitioner() machine_graph, _ = partitioner.__call__(app_graph, machine, 100) allocator = ZonedRoutingInfoAllocator() n_keys_mapper = EdgeToNKeysMapper() n_keys_map = n_keys_mapper.__call__(machine_graph) routing_info = allocator.__call__( machine_graph, n_keys_map, flexible=False) post_vertex = next(iter(post_pop._vertex.machine_vertices)) post_vertex_slice = post_vertex.vertex_slice post_vertex_placement = Placement(post_vertex, 0, 0, 3) temp_spec = tempfile.mktemp() spec = DataSpecificationGenerator(io.FileIO(temp_spec, "wb"), None) synaptic_matrices = SynapticMatrices( post_vertex_slice, n_synapse_types=2, all_single_syn_sz=10000, synaptic_matrix_region=1, direct_matrix_region=2, poptable_region=3, connection_builder_region=4) synaptic_matrices.write_synaptic_data( spec, post_pop._vertex.incoming_projections, all_syn_block_sz=10000, weight_scales=[32, 32], routing_info=routing_info) spec.end_specification() with io.FileIO(temp_spec, "rb") as spec_reader: executor = DataSpecificationExecutor(spec_reader, 20000) executor.execute() all_data = bytearray() all_data.extend(bytearray(executor.get_header())) all_data.extend(bytearray(executor.get_pointer_table(0))) for r in range(MAX_MEM_REGIONS): region = executor.get_region(r) if region is not None: all_data.extend(region.region_data) transceiver = MockTransceiverRawData(all_data) report_folder = mkdtemp() try: connections_1 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_one_to_one_1._projection_edge, proj_one_to_one_1._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_1) == post_vertex_slice.n_atoms assert all([conn["weight"] == 1.5 for conn in connections_1]) assert all([conn["delay"] == 1.0 for conn in connections_1]) connections_2 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_one_to_one_2._projection_edge, proj_one_to_one_2._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_2) == post_vertex_slice.n_atoms assert all([conn["weight"] == 2.5 for conn in connections_2]) assert all([conn["delay"] == 2.0 for conn in connections_2]) connections_3 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_all_to_all._projection_edge, proj_all_to_all._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_3) == 100 assert all([conn["weight"] == 4.5 for conn in connections_3]) assert all([conn["delay"] == 4.0 for conn in connections_3]) connections_4 = numpy.concatenate( synaptic_matrices.get_connections_from_machine( transceiver, post_vertex_placement, proj_from_list._projection_edge, proj_from_list._synapse_information)) # Check that all the connections have the right weight and delay assert len(connections_4) == len(from_list_list) list_weights = [values[2] for values in from_list_list] list_delays = [values[3] for values in from_list_list] assert all(list_weights == connections_4["weight"]) assert all(list_delays == connections_4["delay"]) finally: shutil.rmtree(report_folder, ignore_errors=True)