class TestFileDataWriter(unittest.TestCase):
def setUp(self):
self._dir = os.path.dirname(inspect.getfile(self.__class__))
def _file(self, filename):
return os.path.join(self._dir, "files_data_writer", filename)
def test_write_one_byte(self):
self.writer = FileDataWriter(self._file('txt_one_byte'))
self.writer.write(bytearray([0]))
self.writer.close()
with open(self._file('txt_one_byte'), "r") as file_handle:
self.assertEqual(file_handle.read(1), '\x00')
self.assertEqual(file_handle.read(1), '')
def test_write_five_bytes(self):
self.writer = FileDataWriter(self._file('txt_5_bytes'))
self.writer.write(bytearray([1, 2, 3, 4, 5]))
self.writer.close()
with open(self._file('txt_5_bytes'), "r") as file_handle:
self.assertEqual(file_handle.read(1), '\x01')
self.assertEqual(file_handle.read(1), '\x02')
self.assertEqual(file_handle.read(1), '\x03')
self.assertEqual(file_handle.read(1), '\x04')
self.assertEqual(file_handle.read(1), '\x05')
self.assertEqual(file_handle.read(1), '')
def test_write_from_empty_file(self):
self.writer = FileDataWriter(self._file('txt_empty'))
self.writer.write(bytearray())
self.writer.close()
with open(self._file('txt_empty'), "r") as file_handle:
self.assertEqual(file_handle.read(1), '')
def test_write_synaptic_matrix_and_master_population_table(self):
MockSimulator.setup()
default_config_paths = os.path.join(
os.path.dirname(abstract_spinnaker_common.__file__),
AbstractSpiNNakerCommon.CONFIG_FILE_NAME)
config = conf_loader.load_config(
AbstractSpiNNakerCommon.CONFIG_FILE_NAME, default_config_paths)
config.set("Simulation", "one_to_one_connection_dtcm_max_bytes", 40)
machine_time_step = 1000.0
pre_app_vertex = SimpleApplicationVertex(10)
pre_vertex = SimpleMachineVertex(resources=None)
pre_vertex_slice = Slice(0, 9)
post_app_vertex = SimpleApplicationVertex(10)
post_vertex = SimpleMachineVertex(resources=None)
post_vertex_slice = Slice(0, 9)
post_slice_index = 0
one_to_one_connector_1 = OneToOneConnector(None)
one_to_one_connector_1.set_projection_information(
pre_app_vertex, post_app_vertex, None, machine_time_step)
one_to_one_connector_1.set_weights_and_delays(1.5, 1.0)
one_to_one_connector_2 = OneToOneConnector(None)
one_to_one_connector_2.set_projection_information(
pre_app_vertex, post_app_vertex, None, machine_time_step)
one_to_one_connector_2.set_weights_and_delays(2.5, 2.0)
all_to_all_connector = AllToAllConnector(None)
all_to_all_connector.set_projection_information(
pre_app_vertex, post_app_vertex, None, machine_time_step)
all_to_all_connector.set_weights_and_delays(4.5, 4.0)
direct_synapse_information_1 = SynapseInformation(
one_to_one_connector_1, SynapseDynamicsStatic(), 0)
direct_synapse_information_2 = SynapseInformation(
one_to_one_connector_2, SynapseDynamicsStatic(), 1)
all_to_all_synapse_information = SynapseInformation(
all_to_all_connector, SynapseDynamicsStatic(), 0)
app_edge = ProjectionApplicationEdge(
pre_app_vertex, post_app_vertex, direct_synapse_information_1)
app_edge.add_synapse_information(direct_synapse_information_2)
app_edge.add_synapse_information(all_to_all_synapse_information)
machine_edge = ProjectionMachineEdge(
app_edge.synapse_information, pre_vertex, post_vertex)
partition_name = "TestPartition"
graph = MachineGraph("Test")
graph.add_vertex(pre_vertex)
graph.add_vertex(post_vertex)
graph.add_edge(machine_edge, partition_name)
graph_mapper = GraphMapper()
graph_mapper.add_vertex_mapping(
pre_vertex, pre_vertex_slice, pre_app_vertex)
graph_mapper.add_vertex_mapping(
post_vertex, post_vertex_slice, post_app_vertex)
graph_mapper.add_edge_mapping(machine_edge, app_edge)
weight_scales = [4096.0, 4096.0]
key = 0
routing_info = RoutingInfo()
routing_info.add_partition_info(PartitionRoutingInfo(
[BaseKeyAndMask(key, 0xFFFFFFF0)],
graph.get_outgoing_edge_partition_starting_at_vertex(
pre_vertex, partition_name)))
temp_spec = tempfile.mktemp()
spec_writer = FileDataWriter(temp_spec)
spec = DataSpecificationGenerator(spec_writer, None)
master_pop_sz = 1000
master_pop_region = 0
all_syn_block_sz = 2000
synapse_region = 1
spec.reserve_memory_region(master_pop_region, master_pop_sz)
spec.reserve_memory_region(synapse_region, all_syn_block_sz)
synapse_type = MockSynapseType()
synaptic_manager = SynapticManager(
synapse_type=synapse_type, ring_buffer_sigma=5.0,
spikes_per_second=100.0, config=config)
synaptic_manager._write_synaptic_matrix_and_master_population_table(
spec, [post_vertex_slice], post_slice_index, post_vertex,
post_vertex_slice, all_syn_block_sz, weight_scales,
master_pop_region, synapse_region, routing_info, graph_mapper,
graph, machine_time_step)
spec.end_specification()
spec_writer.close()
spec_reader = FileDataReader(temp_spec)
executor = DataSpecificationExecutor(
spec_reader, master_pop_sz + all_syn_block_sz)
executor.execute()
master_pop_table = executor.get_region(0)
synaptic_matrix = executor.get_region(1)
all_data = bytearray()
all_data.extend(master_pop_table.region_data[
:master_pop_table.max_write_pointer])
all_data.extend(synaptic_matrix.region_data[
:synaptic_matrix.max_write_pointer])
master_pop_table_address = 0
synaptic_matrix_address = master_pop_table.max_write_pointer
direct_synapses_address = struct.unpack_from(
"<I", synaptic_matrix.region_data)[0]
direct_synapses_address += synaptic_matrix_address + 8
indirect_synapses_address = synaptic_matrix_address + 4
placement = Placement(None, 0, 0, 1)
transceiver = MockTransceiverRawData(all_data)
# Get the master population table details
items = synaptic_manager._poptable_type\
.extract_synaptic_matrix_data_location(
key, master_pop_table_address, transceiver,
placement.x, placement.y)
# The first entry should be direct, but the rest should be indirect;
# the second is potentially direct, but has been restricted by the
# restriction on the size of the direct matrix
assert len(items) == 3
# TODO: This has been changed because direct matrices are disabled!
assert not items[0][2]
assert not items[1][2]
assert not items[2][2]
data_1, row_len_1 = synaptic_manager._retrieve_synaptic_block(
transceiver=transceiver, placement=placement,
master_pop_table_address=master_pop_table_address,
indirect_synapses_address=indirect_synapses_address,
direct_synapses_address=direct_synapses_address, key=key,
n_rows=pre_vertex_slice.n_atoms, index=0,
using_extra_monitor_cores=False)
connections_1 = synaptic_manager._synapse_io.read_synapses(
direct_synapse_information_1, pre_vertex_slice, post_vertex_slice,
row_len_1, 0, 2, weight_scales, data_1, None,
app_edge.n_delay_stages, machine_time_step)
# The first matrix is a 1-1 matrix, so row length is 1
assert row_len_1 == 1
# Check that all the connections have the right weight and delay
assert len(connections_1) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 1.5 for conn in connections_1])
assert all([conn["delay"] == 1.0 for conn in connections_1])
data_2, row_len_2 = synaptic_manager._retrieve_synaptic_block(
transceiver=transceiver, placement=placement,
master_pop_table_address=master_pop_table_address,
indirect_synapses_address=indirect_synapses_address,
direct_synapses_address=direct_synapses_address, key=key,
n_rows=pre_vertex_slice.n_atoms, index=1,
using_extra_monitor_cores=False)
connections_2 = synaptic_manager._synapse_io.read_synapses(
direct_synapse_information_2, pre_vertex_slice, post_vertex_slice,
row_len_2, 0, 2, weight_scales, data_2, None,
app_edge.n_delay_stages, machine_time_step)
# The second matrix is a 1-1 matrix, so row length is 1
assert row_len_2 == 1
# Check that all the connections have the right weight and delay
assert len(connections_2) == post_vertex_slice.n_atoms
assert all([conn["weight"] == 2.5 for conn in connections_2])
assert all([conn["delay"] == 2.0 for conn in connections_2])
data_3, row_len_3 = synaptic_manager._retrieve_synaptic_block(
transceiver=transceiver, placement=placement,
master_pop_table_address=master_pop_table_address,
indirect_synapses_address=indirect_synapses_address,
direct_synapses_address=direct_synapses_address, key=key,
n_rows=pre_vertex_slice.n_atoms, index=2,
using_extra_monitor_cores=False)
connections_3 = synaptic_manager._synapse_io.read_synapses(
all_to_all_synapse_information, pre_vertex_slice,
post_vertex_slice, row_len_3, 0, 2, weight_scales, data_3, None,
app_edge.n_delay_stages, machine_time_step)
# The third matrix is an all-to-all matrix, so length is n_atoms
assert row_len_3 == post_vertex_slice.n_atoms
# Check that all the connections have the right weight and delay
assert len(connections_3) == \
post_vertex_slice.n_atoms * pre_vertex_slice.n_atoms
assert all([conn["weight"] == 4.5 for conn in connections_3])
assert all([conn["delay"] == 4.0 for conn in connections_3])
class MyTestCase(unittest.TestCase):
def setUp(self):
self.reader = None
self._dir = os.path.dirname(inspect.getfile(self.__class__))
def _file(self, filename):
return os.path.join(self._dir, "data_files", filename)
def tearDown(self):
if self.reader is not None:
self.reader.close()
def test_one_byte(self):
myfile = self._file('txt_one_byte')
self.writer = FileDataWriter(myfile)
self.writer.write(bytearray([1]))
self.writer.close()
with open(myfile, "rb") as file_handle:
self.assertEqual(file_handle.read(1), '\x01')
self.assertEqual(file_handle.read(1), '')
self.reader = FileDataReader(myfile)
stream = self.reader.read(1)
self.assertEqual(stream[0], '\x01')
def test_readinto_one_byte(self):
myfile = self._file('txt_one_byte')
self.writer = FileDataWriter(myfile)
self.writer.write(bytearray([5]))
self.writer.close()
with open(myfile, "rb") as file_handle:
self.assertEqual(file_handle.read(1), '\x05')
self.assertEqual(file_handle.read(1), '')
self.reader = FileDataReader(myfile)
ba = bytearray(1)
self.reader.readinto(ba)
self.assertEqual(len(ba), 1)
self.assertEqual(ba[0], 5)
def test_read_five_bytes(self):
myfile = self._file('txt_5_bytes')
self.writer = FileDataWriter(myfile)
self.writer.write(bytearray([1, 2, 3, 4, 5]))
self.writer.close()
with open(myfile, "rb") as file_handle:
self.assertEqual(file_handle.read(1), '\x01')
self.assertEqual(file_handle.read(1), '\x02')
self.assertEqual(file_handle.read(1), '\x03')
self.assertEqual(file_handle.read(1), '\x04')
self.assertEqual(file_handle.read(1), '\x05')
self.assertEqual(file_handle.read(1), '')
self.reader = FileDataReader(myfile)
stream = self.reader.read(5)
self.assertEqual(len(stream), 5)
self.assertEqual(stream[0], '\x01')
self.assertEqual(stream[1], '\x02')
self.assertEqual(stream[2], '\x03')
self.assertEqual(stream[3], '\x04')
self.assertEqual(stream[4], '\x05')
def test_read_from_empty_file(self):
myfile = self._file('txt_empty')
self.writer = FileDataWriter(myfile)
self.writer.write(bytearray())
self.writer.close()
with open(myfile, "rb") as file_handle:
self.assertEqual(file_handle.read(1), '')
self.reader = FileDataReader(myfile)
stream = self.reader.read(1)
self.assertEqual(len(stream), 0)
def test_read_truncate(self):
myfile = self._file('txt_one_byte_from_multiple_bytes')
self.writer = FileDataWriter(myfile)
self.writer.write(bytearray([0xF0, 0xA4, 0xAD, 0xA2]))
self.writer.close()
with open(myfile, "rb") as file_handle:
self.assertEqual(file_handle.read(1), '\xf0')
self.assertEqual(file_handle.read(1), '\xA4')
self.assertEqual(file_handle.read(1), '\xAD')
self.assertEqual(file_handle.read(1), '\xA2')
self.assertEqual(file_handle.read(1), '')
self.reader = FileDataReader(myfile)
stream = self.reader.read(2)
self.assertEqual(len(stream), 2)
self.assertEqual(stream[0], '\xf0')
self.assertEqual(stream[1], '\xA4')
def host_based_data_specification_execution(
hostname, transceiver, write_text_specs,
application_data_runtime_folder, machine, report_default_directory,
app_id, dsg_targets):
"""
:param hostname:
:param transceiver:
:param write_text_specs:
:param application_data_runtime_folder:
:param machine:
:param report_default_directory:
:param app_id:
:param dsg_targets:
:return:
"""
processor_to_app_data_base_address = dict()
# create a progress bar for end users
progress_bar = ProgressBar(
len(list(dsg_targets)),
"Executing data specifications and loading data")
for ((x, y, p), data_spec_file_path) in dsg_targets.iteritems():
# build specification reader
data_spec_file_path = dsg_targets[x, y, p]
data_spec_reader = FileDataReader(data_spec_file_path)
# build application data writer
app_data_file_path = \
AbstractDataSpecableVertex.get_application_data_file_path(
x, y, p, hostname, application_data_runtime_folder)
data_writer = FileDataWriter(app_data_file_path)
# generate a file writer for DSE report (app pointer table)
report_writer = None
if write_text_specs:
new_report_directory = os.path.join(report_default_directory,
"data_spec_text_files")
if not os.path.exists(new_report_directory):
os.mkdir(new_report_directory)
file_name = "{}_DSE_report_for_{}_{}_{}.txt".format(
hostname, x, y, p)
report_file_path = os.path.join(new_report_directory,
file_name)
report_writer = FileDataWriter(report_file_path)
# maximum available memory
# however system updates the memory available
# independently, so the check on the space available actually
# happens when memory is allocated
chip = machine.get_chip_at(x, y)
memory_available = chip.sdram.size
# generate data spec executor
host_based_data_spec_executor = DataSpecificationExecutor(
data_spec_reader, data_writer, memory_available, report_writer)
# run data spec executor
try:
# bytes_used_by_spec, bytes_written_by_spec = \
host_based_data_spec_executor.execute()
except exceptions.DataSpecificationException as e:
logger.error(
"Error executing data specification for {}, {}, {}".format(
x, y, p))
raise e
bytes_used_by_spec = \
host_based_data_spec_executor.get_constructed_data_size()
# allocate memory where the app data is going to be written
# this raises an exception in case there is not enough
# SDRAM to allocate
start_address = transceiver.malloc_sdram(x, y, bytes_used_by_spec,
app_id)
# the base address address needs to be passed to the DSE to
# generate the pointer table with absolute addresses
host_based_data_spec_executor.write_dse_output_file(start_address)
# close the application data file writer
data_writer.close()
# the data is written to memory
file_reader = FileDataReader(app_data_file_path)
app_data = file_reader.readall()
bytes_written_by_spec = len(app_data)
transceiver.write_memory(x, y, start_address, app_data)
file_reader.close()
# set user 0 register appropriately to the application data
user_0_address = \
transceiver.get_user_0_register_address_from_core(x, y, p)
start_address_encoded = \
buffer(struct.pack("<I", start_address))
transceiver.write_memory(x, y, user_0_address,
start_address_encoded)
# write information for the memory map report
processor_to_app_data_base_address[x, y, p] = {
'start_address': start_address,
'memory_used': bytes_used_by_spec,
'memory_written': bytes_written_by_spec
}
# update the progress bar
progress_bar.update()
# close the progress bar
progress_bar.end()
return {
'processor_to_app_data_base_address':
processor_to_app_data_base_address,
'LoadedApplicationDataToken': True
}