def __call__(self, placements, file_path):
"""
:param placements: the memory placements object
:param file_path: the file path for the placements.json
:return: file path for the placements.json
"""
# write basic stuff
json_obj = dict()
vertex_by_id = dict()
progress = ProgressBar(placements.n_placements + 1,
"converting to JSON placements")
# process placements
for placement in progress.over(placements, False):
vertex_id = ident(placement.vertex)
vertex_by_id[vertex_id] = placement.vertex
json_obj[vertex_id] = [placement.x, placement.y]
# dump dict into json file
with open(file_path, "w") as file_to_write:
json.dump(json_obj, file_to_write)
progress.update()
# validate the schema
file_format_schemas.validate(json_obj, "placements.json")
progress.end()
# return the file format
return file_path, vertex_by_id
def _ner_route(machine_graph, machine, placements, vector_to_nodes):
""" Performs routing using rig algorithm
:param MachineGraph machine_graph:
:param ~spinn_machine.Machine machine:
:param Placements placements:
:return:
:rtype: MulticastRoutingTableByPartition
"""
routing_tables = MulticastRoutingTableByPartition()
progress_bar = ProgressBar(len(machine_graph.vertices), "Routing")
for source_vertex in progress_bar.over(machine_graph.vertices):
# handle the vertex edges
for partition in machine_graph.\
get_multicast_edge_partitions_starting_at_vertex(
source_vertex):
post_vertexes = list(e.post_vertex for e in partition.edges)
routing_tree = _do_route(source_vertex, post_vertexes, machine,
placements, vector_to_nodes)
incoming_processor = placements.get_placement_of_vertex(
partition.pre_vertex).p
_convert_a_route(routing_tables, partition, incoming_processor,
None, routing_tree)
progress_bar.end()
return routing_tables
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.IDLE)
if error_cores or watchdog_cores or idle_cores:
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(
error_cores.values(), watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
self._update_provenance(txrx, total_processors, processors_completed,
all_core_subsets, app_id, progress)
progress.end()
def __call__(self, machine_graph, machine):
progress_bar = ProgressBar(7, "Placing")
vertices_resources, nets, _ = \
rig_converters.convert_to_rig_graph(machine_graph)
progress_bar.update()
rig_machine = rig_converters.convert_to_rig_machine(machine)
progress_bar.update()
rig_constraints = rig_converters.create_rig_machine_constraints(
machine)
progress_bar.update()
rig_constraints.extend(
rig_converters.create_rig_graph_constraints(
machine_graph, rig_machine))
progress_bar.update()
rig_placements = place(vertices_resources, nets, rig_machine,
rig_constraints)
progress_bar.update()
rig_allocations = allocate(vertices_resources, nets, rig_machine,
rig_constraints, rig_placements)
progress_bar.update()
placements = rig_converters.convert_from_rig_placements(
rig_placements, rig_allocations, machine_graph)
progress_bar.update()
progress_bar.end()
return placements
def __java_sys(self, dsg_targets, executable_targets, region_sizes):
""" Does the Data Specification Execution and loading using Java
:param DataSpecificationTargets dsg_targets:
map of placement to file path
:param ExecutableTargets executable_targets:
the map between binaries and locations and executable types
:param dict(tuple(int,int,int),int) region_sizes:
the coord for region sizes for each core
:return: map of cores to descriptions of what was written
:rtype: DsWriteInfo
"""
# create a progress bar for end users
progress = ProgressBar(
4, "Executing data specifications and loading data for system "
"vertices using Java")
dsg_targets.mark_system_cores(system_cores(executable_targets))
progress.update()
# Copy data from WriteMemoryIOData to database
dw_write_info = self.__java_database(dsg_targets, progress,
region_sizes)
self._java.execute_system_data_specification()
progress.end()
return dw_write_info
def __java_app(self, dsg_targets, executable_targets, use_monitors,
region_sizes):
"""
:param DataSpecificationTargets dsg_targets:
:param ExecutableTargets executable_targets:
:param bool use_monitors:
:param dict(tuple(int,int,int),int) region_sizes:
:return: map of cores to descriptions of what was written
:rtype: DsWriteInfo
"""
# create a progress bar for end users
progress = ProgressBar(
4, "Executing data specifications and loading data for "
"application vertices using Java")
dsg_targets.mark_system_cores(system_cores(executable_targets))
progress.update()
# Copy data from WriteMemoryIOData to database
dw_write_info = self.__java_database(dsg_targets, progress,
region_sizes)
if use_monitors:
self._java.set_placements(self._placements, self._txrx)
self._java.execute_app_data_specification(use_monitors)
progress.end()
return dw_write_info
def __call__(self, machine_graph, machine, file_path):
"""
:param machine_graph: the machine graph
:param machine: the machine
"""
progress = ProgressBar(
machine_graph.n_vertices + 2, "creating JSON constraints")
json_obj = list()
self._add_monitor_core_reserve(json_obj)
progress.update()
self._add_extra_monitor_cores(json_obj, machine)
progress.update()
vertex_by_id = self._search_graph_for_placement_constraints(
json_obj, machine_graph, machine, progress)
with open(file_path, "w") as f:
json.dump(json_obj, f)
# validate the schema
file_format_schemas.validate(json_obj, "constraints.json")
# complete progress bar
progress.end()
return file_path, vertex_by_id
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.IDLE)
if error_cores or watchdog_cores or idle_cores:
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(error_cores.values(),
watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
self._update_provenance(txrx, total_processors, processors_completed,
all_core_subsets, app_id, progress)
progress.end()
def __call__(self, executable_targets, app_id, transceiver,
loaded_application_data_token,
generated_connectors_on_machine_token):
""" Go through the executable targets and load each binary to \
everywhere and then send a start request to the cores that \
actually use it
"""
if not loaded_application_data_token:
raise ConfigurationException(
"The token for having loaded the application data token is set"
" to false and therefore I cannot run. Please fix and try "
"again")
if not generated_connectors_on_machine_token:
raise exceptions.ConfigurationException(
"The token for generating connectors on machine token is set"
" to false and therefore I cannot run. Please fix and try "
"again")
progress = ProgressBar(executable_targets.total_processors + 1,
"Loading executables onto the machine")
for binary in executable_targets.binaries:
progress.update(
self._launch_binary(executable_targets, binary, transceiver,
app_id))
self._start_simulation(executable_targets, transceiver, app_id)
progress.update()
progress.end()
return True
def _write_router_provenance_data(self, router_tables, machine, txrx):
""" Writes the provenance data of the router diagnostics
:param router_tables: the routing tables generated by pacman
:param machine: the spinnMachine object
:param txrx: the transceiver object
"""
progress = ProgressBar(machine.n_chips * 2,
"Getting Router Provenance")
# acquire diagnostic data
items = list()
seen_chips = set()
for router_table in sorted(router_tables.routing_tables,
key=lambda table: (table.x, table.y)):
self._write_router_table_diagnostic(txrx, machine, router_table.x,
router_table.y, seen_chips,
router_table, items)
progress.update()
for chip in sorted(machine.chips, key=lambda c: (c.x, c.y)):
self._write_router_chip_diagnostic(txrx, chip, seen_chips, items)
progress.update()
progress.end()
return items
def __call__(self, nengo_operator_graph, random_number_generator, seed):
"""
:param nengo_operator_graph:
:param random_number_generator:
:param seed:
:return:
"""
progress_bar = ProgressBar(
total_number_of_things_to_do=2,
string_describing_what_being_progressed=(
"Inserting Interposers where suitable."))
# add interposers as required
interposers, interposer_application_graph = \
self._insert_interposers(
nengo_operator_graph, random_number_generator)
progress_bar.update()
# compress interposers when applicable
stacked_interposer_graph = self._stack_interposers(
interposer_application_graph, interposers, random_number_generator,
seed=seed)
progress_bar.update()
progress_bar.end()
# return optimised app graph
return stacked_interposer_graph
def __call__(self,
transceiver,
tags=None,
iptags=None,
reverse_iptags=None):
"""
:param tags: the tags object which contains IP and reverse IP tags.
could be none if these are being given in separate lists
:param iptags: a list of IP tags, given when tags is none
:param reverse_iptags: a list of reverse IP tags when tags is none.
:param transceiver: the transceiver object
"""
# clear all the tags from the Ethernet connection, as nothing should
# be allowed to use it (no two apps should use the same Ethernet
# connection at the same time)
progress = ProgressBar(MAX_TAG_ID, "Clearing tags")
for tag_id in progress.over(range(MAX_TAG_ID)):
transceiver.clear_ip_tag(tag_id)
# Use tags object to supply tag info if it is supplied
if tags is not None:
iptags = list(tags.ip_tags)
reverse_iptags = list(tags.reverse_ip_tags)
# Load the IP tags and the Reverse IP tags
progress = ProgressBar(
len(iptags) + len(reverse_iptags), "Loading Tags")
self.load_iptags(iptags, transceiver, progress)
self.load_reverse_iptags(reverse_iptags, transceiver, progress)
progress.end()
def __call__(self, machine_graph, machine, file_path):
"""
:param machine_graph: the machine graph
:param machine: the machine
"""
progress = ProgressBar(
machine_graph.n_vertices + 2, "creating JSON constraints")
json_obj = list()
self._add_monitor_core_reserve(json_obj)
progress.update()
self._add_extra_monitor_cores(json_obj, machine)
progress.update()
vertex_by_id = self._search_graph_for_placement_constraints(
json_obj, machine_graph, machine, progress)
with open(file_path, "w") as f:
json.dump(json_obj, f)
# validate the schema
file_format_schemas.validate(json_obj, "constraints.json")
# complete progress bar
progress.end()
return file_path, vertex_by_id
def __java_all(self, dsg_targets):
""" Does the Data Specification Execution and loading using Java
:param dsg_targets: map of placement to file path
:type dsg_targets: \
:py:class:`spinn_front_end_common.interface.ds.DataSpecificationTargets`
:return: map of of cores to a dict of \
'start_address', 'memory_used', 'memory_written'
:rtype: spinn_front_end_common.interface.ds.ds_write_info.DsWriteInfo
"""
# create a progress bar for end users
progress = ProgressBar(
3, "Executing data specifications and loading data using Java")
# Copy data from WriteMemoryIOData to database
dw_write_info = DsWriteInfo(dsg_targets.get_database())
dw_write_info.clear_write_info()
if self._write_info_map is not None:
for core, info in iteritems(self._write_info_map):
dw_write_info[core] = info
progress.update()
dsg_targets.set_app_id(self._app_id)
self._java.set_machine(self._machine)
self._java.set_report_folder(self._db_folder)
progress.update()
self._java.execute_data_specification()
progress.end()
return dw_write_info
def __java_app(self, dsg_targets, executable_targets, use_monitors):
# create a progress bar for end users
progress = ProgressBar(
4, "Executing data specifications and loading data for "
"application vertices using Java")
dsg_targets.mark_system_cores(system_cores(executable_targets))
progress.update()
# Copy data from WriteMemoryIOData to database
dw_write_info = DsWriteInfo(dsg_targets.get_database())
dw_write_info.clear_write_info()
if self._write_info_map is not None:
for core, info in iteritems(self._write_info_map):
dw_write_info[core] = info
progress.update()
dsg_targets.set_app_id(self._app_id)
self._java.set_machine(self._machine)
self._java.set_report_folder(self._db_folder)
if use_monitors:
self._java.set_placements(self._placements, self._txrx)
progress.update()
self._java.execute_app_data_specification(use_monitors)
progress.end()
return dw_write_info
def __call__(self, placements, file_path):
"""
:param placements: the memory placements object
:param file_path: the file path for the placements.json
:return: file path for the placements.json
"""
# write basic stuff
json_obj = dict()
vertex_by_id = dict()
progress = ProgressBar(placements.n_placements + 1,
"converting to JSON placements")
# process placements
for placement in progress.over(placements, False):
vertex_id = ident(placement.vertex)
vertex_by_id[vertex_id] = placement.vertex
json_obj[vertex_id] = [placement.x, placement.y]
# dump dict into json file
with open(file_path, "w") as file_to_write:
json.dump(json_obj, file_to_write)
progress.update()
# validate the schema
file_format_schemas.validate(json_obj, "placements.json")
progress.end()
# return the file format
return file_path, vertex_by_id
def __call__(self, machine_graph, machine, plan_n_timesteps, placements):
progress_bar = ProgressBar(7, "Routing")
vertices_resources, nets, net_names = \
convert_to_rig_graph_pure_mc(machine_graph, plan_n_timesteps)
progress_bar.update()
rig_machine = convert_to_rig_machine(machine)
progress_bar.update()
rig_constraints = create_rig_machine_constraints(machine)
progress_bar.update()
rig_constraints.extend(create_rig_graph_constraints(
machine_graph, machine))
progress_bar.update()
rig_placements, rig_allocations = convert_to_rig_placements(
placements, machine)
progress_bar.update()
rig_routes = route(
vertices_resources, nets, rig_machine, rig_constraints,
rig_placements, rig_allocations, "cores")
rig_routes = {
name: rig_routes[net] for net, name in iteritems(net_names)}
progress_bar.update()
routes = convert_from_rig_routes(rig_routes)
progress_bar.update()
progress_bar.end()
return routes
def __call__(self,
transceiver,
tags=None,
iptags=None,
reverse_iptags=None):
"""
:param ~spinnman.transceiver.Transceiver transceiver:
:param ~.Tags tags:
:param list(~spinn_machine.tags.IPTag) iptags:
:param list(~spinn_machine.tags.ReverseIPTag) reverse_iptags:
"""
# clear all the tags from the Ethernet connection, as nothing should
# be allowed to use it (no two apps should use the same Ethernet
# connection at the same time)
progress = ProgressBar(MAX_TAG_ID, "Clearing tags")
for tag_id in progress.over(range(MAX_TAG_ID)):
transceiver.clear_ip_tag(tag_id)
# Use tags object to supply tag info if it is supplied
if tags is not None:
iptags = list(tags.ip_tags)
reverse_iptags = list(tags.reverse_ip_tags)
# Load the IP tags and the Reverse IP tags
progress = ProgressBar(
len(iptags) + len(reverse_iptags), "Loading Tags")
self.load_iptags(iptags, transceiver, progress)
self.load_reverse_iptags(reverse_iptags, transceiver, progress)
progress.end()
def __call__(self, placements, file_path):
"""
:param placements:
:param file_path:
"""
progress = ProgressBar(len(placements) + 1,
"Converting to JSON core allocations")
# write basic stuff
json_obj = OrderedDict()
json_obj['type'] = "cores"
vertex_by_id = OrderedDict()
# process placements
for placement in progress.over(placements, False):
self._convert_placement(placement, vertex_by_id, json_obj)
# dump dict into json file
with open(file_path, "w") as f:
json.dump(json_obj, f)
progress.update()
# validate the schema
file_format_schemas.validate(json_obj, "core_allocations.json")
# complete progress bar
progress.end()
# return the file format
return file_path, vertex_by_id
def __call__(
self, transceiver, tags=None, iptags=None, reverse_iptags=None):
"""
:param tags: the tags object which contains IP and reverse IP tags.
could be none if these are being given in separate lists
:param iptags: a list of IP tags, given when tags is none
:param reverse_iptags: a list of reverse IP tags when tags is none.
:param transceiver: the transceiver object
"""
# clear all the tags from the Ethernet connection, as nothing should
# be allowed to use it (no two apps should use the same Ethernet
# connection at the same time)
progress = ProgressBar(MAX_TAG_ID, "Clearing tags")
for tag_id in progress.over(range(MAX_TAG_ID)):
transceiver.clear_ip_tag(tag_id)
# Use tags object to supply tag info if it is supplied
if tags is not None:
iptags = list(tags.ip_tags)
reverse_iptags = list(tags.reverse_ip_tags)
# Load the IP tags and the Reverse IP tags
progress = ProgressBar(
len(iptags) + len(reverse_iptags), "Loading Tags")
self.load_iptags(iptags, transceiver, progress)
self.load_reverse_iptags(reverse_iptags, transceiver, progress)
progress.end()
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.IDLE)
if (len(error_cores) != 0 or len(watchdog_cores) != 0
or len(idle_cores) != 0):
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(error_cores.values(),
watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
attempts = 0
while processors_completed != total_processors and attempts < 10:
attempts += 1
unsuccessful_cores = txrx.get_cores_not_in_state(
all_core_subsets, CPUState.FINISHED)
for (x, y, p) in unsuccessful_cores.iterkeys():
data = struct.pack(
"<I", SDP_RUNNING_MESSAGE_CODES.
SDP_UPDATE_PROVENCE_REGION_AND_EXIT.value)
txrx.send_sdp_message(
SDPMessage(SDPHeader(flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_cpu=p,
destination_chip_x=x,
destination_port=SDP_PORTS.
RUNNING_COMMAND_SDP_PORT.value,
destination_chip_y=y),
data=data))
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
left_over_now = total_processors - processors_completed
to_update = left_to_do_cores - left_over_now
left_to_do_cores = left_over_now
if to_update != 0:
progress.update(to_update)
if attempts >= 10:
logger.error("Unable to Finish getting provenance data. "
"Abandoned after too many retries. "
"Board may be left in an unstable state!")
progress.end()
def __call__(self, machine_graph, machine, plan_n_timesteps):
progress_bar = ProgressBar(7, "Placing")
vertices_resources, nets, _ = \
convert_to_rig_graph(machine_graph, plan_n_timesteps)
progress_bar.update()
rig_machine = convert_to_rig_machine(machine)
progress_bar.update()
rig_constraints = create_rig_machine_constraints(machine)
progress_bar.update()
rig_constraints.extend(create_rig_graph_constraints(
machine_graph, rig_machine))
progress_bar.update()
rig_placements = place(
vertices_resources, nets, rig_machine, rig_constraints)
progress_bar.update()
rig_allocations = allocate(
vertices_resources, nets, rig_machine, rig_constraints,
rig_placements)
progress_bar.update()
placements = convert_from_rig_placements(
rig_placements, rig_allocations, machine_graph)
progress_bar.update()
progress_bar.end()
return placements
def __call__(self, machine_graph, file_path):
"""
:param machine_graph: The graph to convert
:param file_path: Where to write the JSON
"""
progress = ProgressBar(machine_graph.n_vertices + 1,
"Converting to JSON graph")
# write basic stuff
json_graph = OrderedDict()
# write vertices data
vertices = OrderedDict()
json_graph["vertices_resources"] = vertices
edges = OrderedDict()
json_graph["edges"] = edges
vertex_by_id = OrderedDict()
partition_by_id = OrderedDict()
for vertex in progress.over(machine_graph.vertices, False):
self._convert_vertex(vertex, vertex_by_id, vertices, edges,
machine_graph, partition_by_id)
with open(file_path, "w") as file_to_write:
json.dump(json_graph, file_to_write)
progress.update()
file_format_schemas.validate(json_graph, "machine_graph.json")
progress.end()
return file_path, vertex_by_id, partition_by_id
def synapse_expander(
app_graph, graph_mapper, placements, transceiver,
provenance_file_path, executable_finder):
""" Run the synapse expander - needs to be done after data has been loaded
"""
synapse_expander = executable_finder.get_executable_path(SYNAPSE_EXPANDER)
delay_expander = executable_finder.get_executable_path(DELAY_EXPANDER)
progress = ProgressBar(len(app_graph.vertices) + 2, "Expanding Synapses")
# Find the places where the synapse expander and delay receivers should run
expander_cores = ExecutableTargets()
for vertex in progress.over(app_graph.vertices, finish_at_end=False):
# Find population vertices
if isinstance(
vertex, (AbstractPopulationVertex, DelayExtensionVertex)):
# Add all machine vertices of the population vertex to ones
# that need synapse expansion
for m_vertex in graph_mapper.get_machine_vertices(vertex):
vertex_slice = graph_mapper.get_slice(m_vertex)
if vertex.gen_on_machine(vertex_slice):
placement = placements.get_placement_of_vertex(m_vertex)
if isinstance(vertex, AbstractPopulationVertex):
binary = synapse_expander
else:
binary = delay_expander
expander_cores.add_processor(
binary, placement.x, placement.y, placement.p)
# Launch the delay receivers
expander_app_id = transceiver.app_id_tracker.get_new_id()
transceiver.execute_application(expander_cores, expander_app_id)
progress.update()
# Wait for everything to finish
finished = False
try:
transceiver.wait_for_cores_to_be_in_state(
expander_cores.all_core_subsets, expander_app_id,
[CPUState.FINISHED])
progress.update()
finished = True
_extract_iobuf(expander_cores, transceiver, provenance_file_path)
progress.end()
except Exception:
logger.exception("Synapse expander has failed")
_handle_failure(
expander_cores, transceiver, provenance_file_path)
finally:
transceiver.stop_application(expander_app_id)
transceiver.app_id_tracker.free_id(expander_app_id)
if not finished:
raise SpynnakerException(
"The synapse expander failed to complete")
def __call__(self, placements, app_graph, executable_finder, transceiver,
machine_graph, routing_infos):
""" Loads and runs the bit field generator on chip.
:param ~pacman.model.placements.Placements placements: placements
:param ~pacman.model.graphs.application.ApplicationGraph app_graph:
the app graph
:param executable_finder: the executable finder
:type executable_finder:
~spinn_front_end_common.utilities.utility_objs.ExecutableFinder
:param ~spinnman.transceiver.Transceiver transceiver:
the SpiNNMan instance
:param ~pacman.model.graphs.machine.MachineGraph machine_graph:
the machine graph
:param ~pacman.model.routing_info.RoutingInfo routing_infos:
the key to edge map
"""
self.__txrx = transceiver
self.__placements = placements
self.__aplx = executable_finder.get_executable_path(
self._BIT_FIELD_EXPANDER_APLX)
# progress bar
progress = ProgressBar(
app_graph.n_vertices + machine_graph.n_vertices + 1,
"Running bitfield generation on chip")
# get data
expander_cores = self._calculate_core_data(app_graph, progress)
# load data
bit_field_app_id = transceiver.app_id_tracker.get_new_id()
progress.update(1)
# run app
system_control_logic.run_system_application(
expander_cores,
bit_field_app_id,
transceiver,
executable_finder,
get_config_bool("Reports", "write_bit_field_iobuf"),
self.__check_for_success, [CPUState.FINISHED],
False,
"bit_field_expander_on_{}_{}_{}.txt",
progress_bar=progress)
# update progress bar
progress.end()
# read in bit fields for debugging purposes
if get_config_bool("Reports", "generate_bit_field_report"):
self._full_report_bit_fields(
app_graph,
os.path.join(report_default_directory(),
self._BIT_FIELD_REPORT_FILENAME))
self._summary_report_bit_fields(
app_graph,
os.path.join(report_default_directory(),
self._BIT_FIELD_SUMMARY_REPORT_FILENAME))
def synapse_expander(
app_graph, graph_mapper, placements, transceiver,
provenance_file_path, executable_finder):
""" Run the synapse expander - needs to be done after data has been loaded
"""
synapse_expander = executable_finder.get_executable_path(SYNAPSE_EXPANDER)
delay_expander = executable_finder.get_executable_path(DELAY_EXPANDER)
progress = ProgressBar(len(app_graph.vertices) + 2, "Expanding Synapses")
# Find the places where the synapse expander and delay receivers should run
expander_cores = ExecutableTargets()
for vertex in progress.over(app_graph.vertices, finish_at_end=False):
# Find population vertices
if isinstance(
vertex, (AbstractPopulationVertex, DelayExtensionVertex)):
# Add all machine vertices of the population vertex to ones
# that need synapse expansion
for m_vertex in graph_mapper.get_machine_vertices(vertex):
vertex_slice = graph_mapper.get_slice(m_vertex)
if vertex.gen_on_machine(vertex_slice):
placement = placements.get_placement_of_vertex(m_vertex)
if isinstance(vertex, AbstractPopulationVertex):
binary = synapse_expander
else:
binary = delay_expander
expander_cores.add_processor(
binary, placement.x, placement.y, placement.p)
# Launch the delay receivers
expander_app_id = transceiver.app_id_tracker.get_new_id()
transceiver.execute_application(expander_cores, expander_app_id)
progress.update()
# Wait for everything to finish
finished = False
try:
transceiver.wait_for_cores_to_be_in_state(
expander_cores.all_core_subsets, expander_app_id,
[CPUState.FINISHED])
progress.update()
finished = True
_extract_iobuf(expander_cores, transceiver, provenance_file_path)
progress.end()
except Exception:
logger.exception("Synapse expander has failed")
_handle_failure(
expander_cores, transceiver, provenance_file_path)
finally:
transceiver.stop_application(expander_app_id)
transceiver.app_id_tracker.free_id(expander_app_id)
if not finished:
raise SpynnakerException(
"The synapse expander failed to complete")
def __call__(self, app_id, txrx, executable_types):
"""
:param int app_id:
:param ~spinnman.transceiver.Transceiver txrx:
:param dict(ExecutableType,~spinn_machine.CoreSubsets) \
executable_types:
:raises ExecutableFailedToStopException:
"""
total_processors = \
len(executable_types[ExecutableType.USES_SIMULATION_INTERFACE])
all_core_subsets = \
executable_types[ExecutableType.USES_SIMULATION_INTERFACE]
progress = ProgressBar(
total_processors,
"Turning off all the cores within the simulation")
# check that the right number of processors are finished
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
finished_cores = processors_finished
while processors_finished != total_processors:
if processors_finished > finished_cores:
progress.update(processors_finished - finished_cores)
finished_cores = processors_finished
processors_rte = txrx.get_core_state_count(
app_id, CPUState.RUN_TIME_EXCEPTION)
processors_watchdogged = txrx.get_core_state_count(
app_id, CPUState.WATCHDOG)
if processors_rte > 0 or processors_watchdogged > 0:
raise ExecutableFailedToStopException(
"{} of {} processors went into an error state when"
" shutting down".format(
processors_rte + processors_watchdogged,
total_processors))
successful_cores_finished = txrx.get_cores_in_state(
all_core_subsets, CPUState.FINISHED)
for core_subset in all_core_subsets:
for processor in core_subset.processor_ids:
if not successful_cores_finished.is_core(
core_subset.x, core_subset.y, processor):
self._update_provenance_and_exit(
txrx, processor, core_subset)
time.sleep(0.5)
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
progress.end()
def __call__(self, machine_graph, n_keys_map, graph_mapper=None):
# check that this algorithm supports the constraints
check_algorithm_can_support_constraints(
constrained_vertices=machine_graph.outgoing_edge_partitions,
supported_constraints=[
FixedMaskConstraint,
FixedKeyAndMaskConstraint,
ContiguousKeyRangeContraint, ShareKeyConstraint],
abstract_constraint_type=AbstractKeyAllocatorConstraint)
# verify that no edge has more than 1 of a constraint ,and that
# constraints are compatible
check_types_of_edge_constraint(machine_graph)
# final keys allocations
routing_infos = RoutingInfo()
# Get the edges grouped by those that require the same key
(fixed_keys, shared_keys, fixed_masks, fixed_fields, flexi_fields,
continuous, noncontinuous) = get_edge_groups(
machine_graph, EdgeTrafficType.MULTICAST)
# Go through the groups and allocate keys
progress = ProgressBar(
machine_graph.n_outgoing_edge_partitions,
"Allocating routing keys")
# allocate the groups that have fixed keys
for group in progress.over(fixed_keys, False):
self._allocate_fixed_keys(group, routing_infos)
for group in progress.over(fixed_masks, False):
self._allocate_fixed_masks(group, n_keys_map, routing_infos)
for group in progress.over(fixed_fields, False):
self._allocate_fixed_fields(group, n_keys_map, routing_infos)
if flexi_fields:
raise PacmanConfigurationException(
"MallocBasedRoutingInfoAllocator does not support FlexiField")
for group in progress.over(shared_keys, False):
self._allocate_share_key(group, routing_infos, n_keys_map)
for group in continuous:
self._allocate_other_groups(group, routing_infos, n_keys_map,
continuous=True)
for group in noncontinuous:
self._allocate_other_groups(group, routing_infos, n_keys_map,
continuous=False)
progress.end()
return routing_infos
def __call__(self, machine_graph, n_keys_map):
"""
:param MachineGraph machine_graph:
:param AbstractMachinePartitionNKeysMap n_keys_map:
:rtype: RoutingInfo
:raises PacmanRouteInfoAllocationException:
"""
self._n_keys_map = n_keys_map
# check that this algorithm supports the constraints
check_algorithm_can_support_constraints(
constrained_vertices=machine_graph.outgoing_edge_partitions,
supported_constraints=[
FixedMaskConstraint, FixedKeyAndMaskConstraint,
ContiguousKeyRangeContraint, ShareKeyConstraint
],
abstract_constraint_type=AbstractKeyAllocatorConstraint)
# verify that no edge has more than 1 of a constraint ,and that
# constraints are compatible
check_types_of_edge_constraint(machine_graph)
# final keys allocations
routing_infos = RoutingInfo()
# Get the edges grouped by those that require the same key
(fixed_keys, shared_keys, fixed_masks, fixed_fields, continuous,
noncontinuous) = get_mulitcast_edge_groups(machine_graph)
# Go through the groups and allocate keys
progress = ProgressBar(machine_graph.n_outgoing_edge_partitions,
"Allocating routing keys")
# allocate the groups that have fixed keys
for group in progress.over(fixed_keys, False):
self._allocate_fixed_keys(group, routing_infos)
for group in progress.over(fixed_masks, False):
self._allocate_fixed_masks(group, routing_infos)
for group in progress.over(fixed_fields, False):
self._allocate_fixed_fields(group, routing_infos)
for group in progress.over(shared_keys, False):
self._allocate_share_key(group, routing_infos)
for group in continuous:
self._allocate_other_groups(group, routing_infos, True)
for group in noncontinuous:
self._allocate_other_groups(group, routing_infos, False)
progress.end()
return routing_infos
def __call__(self, machine_graph, n_keys_map, graph_mapper=None):
# check that this algorithm supports the constraints
check_algorithm_can_support_constraints(
constrained_vertices=machine_graph.outgoing_edge_partitions,
supported_constraints=[
FixedMaskConstraint, FixedKeyAndMaskConstraint,
ContiguousKeyRangeContraint, ShareKeyConstraint
],
abstract_constraint_type=AbstractKeyAllocatorConstraint)
# verify that no edge has more than 1 of a constraint ,and that
# constraints are compatible
utilities.check_types_of_edge_constraint(machine_graph)
# final keys allocations
routing_infos = RoutingInfo()
# Get the edges grouped by those that require the same key
(fixed_keys, shared_keys, fixed_masks, fixed_fields, flexi_fields,
continuous,
noncontinuous) = utilities.get_edge_groups(machine_graph,
EdgeTrafficType.MULTICAST)
# Even non-continuous keys will be continuous
for group in noncontinuous:
continuous.append(group)
# Go through the groups and allocate keys
progress = ProgressBar(machine_graph.n_outgoing_edge_partitions,
"Allocating routing keys")
# allocate the groups that have fixed keys
for group in progress.over(fixed_keys, False):
self._allocate_fixed_keys(group, routing_infos)
for group in progress.over(fixed_masks, False):
self._allocate_fixed_masks(group, n_keys_map, routing_infos)
for group in progress.over(fixed_fields, False):
self._allocate_fixed_fields(group, n_keys_map, routing_infos)
if flexi_fields:
raise PacmanConfigurationException(
"MallocBasedRoutingInfoAllocator does not support FlexiField")
for group in progress.over(shared_keys, False):
self._allocate_share_key(group, routing_infos, n_keys_map)
for group in continuous:
self._allocate_continuous_groups(group, routing_infos, n_keys_map)
progress.end()
return routing_infos
def __call__(self, executable_targets, app_id, transceiver):
progress = ProgressBar(
executable_targets.total_processors + 1,
"Loading executables onto the machine")
for binary in executable_targets.binaries:
progress.update(self._launch_binary(
executable_targets, binary, transceiver, app_id))
self._start_simulation(executable_targets, transceiver, app_id)
progress.update()
progress.end()
def __call__(self, executable_targets, app_id, transceiver):
progress = ProgressBar(executable_targets.total_processors + 1,
"Loading executables onto the machine")
for binary in executable_targets.binaries:
progress.update(
self._launch_binary(executable_targets, binary, transceiver,
app_id))
self._start_simulation(executable_targets, transceiver, app_id)
progress.update()
progress.end()
def __call__(self, machine_graph, placements, buffer_manager):
# Count the regions to be read
n_regions_to_read, vertices = self._count_regions(machine_graph)
# Read back the regions
progress = ProgressBar(
n_regions_to_read, "Extracting buffers from the last run")
try:
buffer_manager.get_data_for_vertices(vertices, progress)
finally:
progress.end()
def __call__(self, machine_graph, machine, plan_n_timesteps):
"""
:param machine_graph: The machine_graph to place
:type machine_graph:\
:py:class:`pacman.model.graphs.machine.MachineGraph`
:param machine:\
The machine with respect to which to partition the application\
graph
:type machine: :py:class:`spinn_machine.Machine`
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:return: A set of placements
:rtype: :py:class:`pacman.model.placements.Placements`
:raise pacman.exceptions.PacmanPlaceException: \
If something goes wrong with the placement
"""
# check that the algorithm can handle the constraints
self._check_constraints(machine_graph.vertices)
# Sort the vertices into those with and those without
# placement constraints
placements = Placements()
constrained = list()
unconstrained = set()
for vertex in machine_graph.vertices:
if locate_constraints_of_type(
vertex.constraints, AbstractPlacerConstraint):
constrained.append(vertex)
else:
unconstrained.add(vertex)
# Iterate over constrained vertices and generate placements
progress = ProgressBar(
machine_graph.n_vertices, "Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_radial_chips(machine))
constrained = sort_vertices_by_known_constraints(constrained)
for vertex in progress.over(constrained, False):
self._place_vertex(vertex, resource_tracker, machine, placements)
while unconstrained:
# Place the subgraph with the overall most connected vertex
max_connected_vertex = self._find_max_connected_vertex(
unconstrained, machine_graph)
self._place_unconstrained_subgraph(
max_connected_vertex, machine_graph, unconstrained,
machine, placements, resource_tracker, progress)
# finished, so stop progress bar and return placements
progress.end()
return placements
def load_initial_buffers(self):
""" Load the initial buffers for the senders using memory writes.
"""
total_data = 0
for vertex in self._sender_vertices:
for region in vertex.get_regions():
total_data += vertex.get_region_buffer_size(region)
progress = ProgressBar(total_data, "Loading buffers")
for vertex in self._sender_vertices:
for region in vertex.get_regions():
self._send_initial_messages(vertex, region, progress)
progress.end()
def _run(self, machine_graph, machine, plan_n_timesteps):
"""
:param MachineGraph machine_graph: The machine_graph to place
:param ~spinn_machine.Machine machine:
The machine with respect to which to partition the application
graph
:param int plan_n_timesteps: number of timesteps to plan for
:return: A set of placements
:rtype: ~pacman.model.placements.Placements
:raise PacmanPlaceException:
If something goes wrong with the placement
"""
# check that the algorithm can handle the constraints
self._check_constraints(machine_graph.vertices)
# Sort the vertices into those with and those without
# placement constraints
placements = Placements()
constrained = list()
unconstrained = set()
for vertex in machine_graph.vertices:
if locate_constraints_of_type(vertex.constraints,
AbstractPlacerConstraint):
constrained.append(vertex)
else:
unconstrained.add(vertex)
# Iterate over constrained vertices and generate placements
progress = ProgressBar(machine_graph.n_vertices,
"Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_radial_chips(machine))
constrained = sort_vertices_by_known_constraints(constrained)
vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
for vertex in progress.over(constrained, False):
self._place_vertex(vertex, resource_tracker, machine, placements,
vertices_on_same_chip, machine_graph)
while unconstrained:
# Place the subgraph with the overall most connected vertex
max_connected_vertex = self._find_max_connected_vertex(
unconstrained, machine_graph)
self._place_unconstrained_subgraph(max_connected_vertex,
machine_graph, unconstrained,
machine, placements,
resource_tracker, progress,
vertices_on_same_chip)
# finished, so stop progress bar and return placements
progress.end()
return placements
def __load_images(self, executable_targets, app_id, txrx, filt, label):
# Compute what work is to be done here
binaries, cores = self.__filter(executable_targets, filt)
# ISSUE: Loading order may be non-constant on older Python
progress = ProgressBar(cores.total_processors + 1, label)
for binary in binaries:
progress.update(
flood_fill_binary_to_spinnaker(executable_targets, binary,
txrx, app_id))
self.__start_simulation(cores, txrx, app_id)
progress.update()
progress.end()
def __call__(self, machine_graph, placements, buffer_manager):
# Count the regions to be read
n_regions_to_read, recording_placements = self._count_regions(
machine_graph, placements)
# Read back the regions
progress = ProgressBar(
n_regions_to_read, "Extracting buffers from the last run")
try:
buffer_manager.get_data_for_placements(
recording_placements, progress)
finally:
progress.end()
def read_samples(self, buffer_manager):
""" Read back the samples
"""
progress = ProgressBar(len(self._mcmc_placements), "Reading results")
samples = list()
for placement in self._mcmc_placements:
# Read the data recorded
sample = placement.vertex.read_samples(buffer_manager, placement)
if sample is not None:
samples.append(sample)
progress.update()
progress.end()
# Merge all the arrays
return numpy.hstack(samples)
def __call__(self, app_id, txrx, executable_targets, has_ran):
if not has_ran:
raise exceptions.ConfigurationException(
"The ran token is not set correctly, please fix and try again")
total_processors = executable_targets.total_processors
all_core_subsets = executable_targets.all_core_subsets
progress = ProgressBar(
total_processors,
"Turning off all the cores within the simulation")
# check that the right number of processors are finished
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
finished_cores = processors_finished
while processors_finished != total_processors:
if processors_finished > finished_cores:
progress.update(finished_cores - processors_finished)
finished_cores = processors_finished
processors_rte = txrx.get_core_state_count(
app_id, CPUState.RUN_TIME_EXCEPTION)
processors_watchdogged = txrx.get_core_state_count(
app_id, CPUState.WATCHDOG)
if processors_rte > 0 or processors_watchdogged > 0:
raise exceptions.ExecutableFailedToStopException(
"{} of {} processors went into an error state when"
" shutting down".format(
processors_rte + processors_watchdogged,
total_processors))
successful_cores_finished = txrx.get_cores_in_state(
all_core_subsets, CPUState.FINISHED)
for core_subset in all_core_subsets:
for processor in core_subset.processor_ids:
if not successful_cores_finished.is_core(
core_subset.x, core_subset.y, processor):
self._update_provenance_and_exit(
txrx, processor, core_subset)
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
progress.end()
def __call__(self, app_id, txrx, executable_types):
total_processors = \
len(executable_types[ExecutableType.USES_SIMULATION_INTERFACE])
all_core_subsets = \
executable_types[ExecutableType.USES_SIMULATION_INTERFACE]
progress = ProgressBar(
total_processors,
"Turning off all the cores within the simulation")
# check that the right number of processors are finished
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
finished_cores = processors_finished
while processors_finished != total_processors:
if processors_finished > finished_cores:
progress.update(processors_finished - finished_cores)
finished_cores = processors_finished
processors_rte = txrx.get_core_state_count(
app_id, CPUState.RUN_TIME_EXCEPTION)
processors_watchdogged = txrx.get_core_state_count(
app_id, CPUState.WATCHDOG)
if processors_rte > 0 or processors_watchdogged > 0:
raise ExecutableFailedToStopException(
"{} of {} processors went into an error state when"
" shutting down".format(
processors_rte + processors_watchdogged,
total_processors))
successful_cores_finished = txrx.get_cores_in_state(
all_core_subsets, CPUState.FINISHED)
for core_subset in all_core_subsets:
for processor in core_subset.processor_ids:
if not successful_cores_finished.is_core(
core_subset.x, core_subset.y, processor):
self._update_provenance_and_exit(
txrx, processor, core_subset)
time.sleep(0.5)
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
progress.end()
def __call__(self, report_folder, application_graph):
"""
:param report_folder: the report folder to put figure into
:param application_graph: the app graph
:rtype: None
"""
# create holders for data
vertex_holders = dict()
dot_diagram = self._get_diagram(
"The graph of the network in graphical form")
# build progress bar for the vertices, edges, and rendering
progress = ProgressBar(
application_graph.n_vertices +
application_graph.n_outgoing_edge_partitions + 1,
"generating the graphical representation of the neural network")
# write vertices into dot diagram
for vertex_counter, vertex in progress.over(
enumerate(application_graph.vertices), False):
dot_diagram.node(
"{}".format(vertex_counter),
"{} ({} neurons)".format(vertex.label, vertex.n_atoms))
vertex_holders[vertex] = vertex_counter
# write edges into dot diagram
for partition in progress.over(
application_graph.outgoing_edge_partitions, False):
for edge in partition.edges:
source_vertex_id = vertex_holders[edge.pre_vertex]
dest_vertex_id = vertex_holders[edge.post_vertex]
if isinstance(edge, ProjectionApplicationEdge):
for synapse_info in edge.synapse_information:
dot_diagram.edge(
"{}".format(source_vertex_id),
"{}".format(dest_vertex_id),
"{}".format(synapse_info.connector))
else:
dot_diagram.edge(
"{}".format(source_vertex_id),
"{}".format(dest_vertex_id))
# write dot file and generate pdf
file_to_output = os.path.join(report_folder, "network_graph.gv")
dot_diagram.render(file_to_output, view=False)
progress.update()
progress.end()
def __call__(self, report_folder, application_graph):
"""
:param report_folder: the report folder to put figure into
:param application_graph: the app graph
:rtype: None
"""
# create holders for data
vertex_holders = dict()
dot_diagram = self._get_diagram(
"The graph of the network in graphical form")
# build progress bar for the vertices, edges, and rendering
progress = ProgressBar(
application_graph.n_vertices +
application_graph.n_outgoing_edge_partitions + 1,
"generating the graphical representation of the neural network")
# write vertices into dot diagram
for vertex_counter, vertex in progress.over(
enumerate(application_graph.vertices), False):
dot_diagram.node(
"{}".format(vertex_counter),
"{} ({} neurons)".format(vertex.label, vertex.n_atoms))
vertex_holders[vertex] = vertex_counter
# write edges into dot diagram
for partition in progress.over(
application_graph.outgoing_edge_partitions, False):
for edge in partition.edges:
source_vertex_id = vertex_holders[edge.pre_vertex]
dest_vertex_id = vertex_holders[edge.post_vertex]
if isinstance(edge, ProjectionApplicationEdge):
for synapse_info in edge.synapse_information:
dot_diagram.edge(
"{}".format(source_vertex_id),
"{}".format(dest_vertex_id),
"{}".format(synapse_info.connector))
else:
dot_diagram.edge(
"{}".format(source_vertex_id),
"{}".format(dest_vertex_id))
# write dot file and generate pdf
file_to_output = os.path.join(report_folder, "network_graph.gv")
dot_diagram.render(file_to_output, view=False)
progress.update()
progress.end()
def __call__(
self, machine_graph, application_graph=None,
provenance_data_objects=None):
"""
:param machine_graph: The machine graph to inspect
:param application_graph: The optional application graph
:param provenance_data_objects: Any existing objects to append to
"""
if provenance_data_objects is not None:
prov_items = provenance_data_objects
else:
prov_items = list()
progress = ProgressBar(
machine_graph.n_vertices +
machine_graph.n_outgoing_edge_partitions,
"Getting provenance data from machine graph")
for vertex in machine_graph.vertices:
if isinstance(vertex, AbstractProvidesLocalProvenanceData):
prov_items.extend(vertex.get_local_provenance_data())
progress.update()
for partition in machine_graph.outgoing_edge_partitions:
for edge in partition.edges:
if isinstance(edge, AbstractProvidesLocalProvenanceData):
prov_items.extend(edge.get_local_provenance_data())
progress.update()
progress.end()
if application_graph is not None:
progress = ProgressBar(
application_graph.n_vertices +
application_graph.n_outgoing_edge_partitions,
"Getting provenance data from application graph")
for vertex in application_graph.vertices:
if isinstance(vertex, AbstractProvidesLocalProvenanceData):
prov_items.extend(vertex.get_local_provenance_data())
progress.update()
for partition in application_graph.outgoing_edge_partitions:
for edge in partition.edges:
if isinstance(edge, AbstractProvidesLocalProvenanceData):
prov_items.extend(edge.get_local_provenance_data())
progress.update()
progress.end()
return prov_items
def __call__(self, machine, file_path):
"""
:param machine:
:param file_path:
"""
progress = ProgressBar(
(machine.max_chip_x + 1) * (machine.max_chip_y + 1) + 2,
"Converting to JSON machine")
# write basic stuff
json_obj = {
"width": machine.max_chip_x + 1,
"height": machine.max_chip_y + 1,
"chip_resources": {
"cores": CHIP_HOMOGENEOUS_CORES,
"sdram": CHIP_HOMOGENEOUS_SDRAM,
"sram": CHIP_HOMOGENEOUS_SRAM,
"router_entries": ROUTER_HOMOGENEOUS_ENTRIES,
"tags": CHIP_HOMOGENEOUS_TAGS},
"dead_chips": [],
"dead_links": []}
# handle exceptions (dead chips)
exceptions = defaultdict(dict)
for x in range(0, machine.max_chip_x + 1):
for y in progress.over(range(0, machine.max_chip_y + 1), False):
self._add_exceptions(json_obj, machine, x, y, exceptions)
json_obj["chip_resource_exceptions"] = [
[x, y, exceptions[x, y]] for x, y in exceptions]
progress.update()
# dump to json file
with open(file_path, "w") as f:
json.dump(json_obj, f)
progress.update()
# validate the schema
file_format_schemas.validate(json_obj, "machine.json")
# update and complete progress bar
progress.end()
return file_path
def __call__(self, machine_graph, placements, machine,
vertex_to_ethernet_connected_chip_mapping,
application_graph=None, graph_mapper=None):
"""
:param machine_graph: the machine graph instance
:param placements: the placements
:param machine: the machine object
:param application_graph: the application graph
:param vertex_to_ethernet_connected_chip_mapping: \
mapping between ethernet connected chips and packet gatherers
:param graph_mapper: the graph mapper
:rtype: None
"""
# pylint: disable=too-many-arguments
n_app_vertices = 0
if application_graph is not None:
n_app_vertices = application_graph.n_vertices
progress = ProgressBar(
machine_graph.n_vertices + n_app_vertices,
"Inserting edges between vertices which require FR speed up "
"functionality.")
for vertex in progress.over(machine_graph.vertices, False):
if isinstance(vertex, ExtraMonitorSupportMachineVertex):
self._process_vertex(
vertex, machine, placements, machine_graph,
vertex_to_ethernet_connected_chip_mapping,
application_graph, graph_mapper)
if application_graph is not None:
for vertex in progress.over(application_graph.vertices, False):
if isinstance(vertex, ExtraMonitorSupport):
machine_verts = graph_mapper.get_machine_vertices(vertex)
for machine_vertex in machine_verts:
self._process_vertex(
machine_vertex, machine, placements, machine_graph,
vertex_to_ethernet_connected_chip_mapping,
application_graph, graph_mapper)
progress.end()
class UpdateRuntimeProcess(AbstractMultiConnectionProcess):
def __init__(self, connection_selector):
super(UpdateRuntimeProcess, self).__init__(connection_selector)
self._progress = None
def receive_response(self, response): # @UnusedVariable
if self._progress is not None:
self._progress.update()
def update_runtime(self, run_time, infinite_run, core_subsets, n_cores):
self._progress = ProgressBar(n_cores, "Updating run time")
for core_subset in core_subsets:
for processor_id in core_subset.processor_ids:
self._send_request(
SCPUpdateRuntimeRequest(
core_subset.x, core_subset.y, processor_id,
run_time, infinite_run,
SDP_PORTS.RUNNING_COMMAND_SDP_PORT.value),
callback=self.receive_response)
self._finish()
self._progress.end()
self.check_for_error()
class ClearIOBUFProcess(AbstractMultiConnectionProcess):
def __init__(self, connection_selector):
super(ClearIOBUFProcess, self).__init__(connection_selector)
self._progress = None
def receive_response(self, response): # @UnusedVariable
if self._progress is not None:
self._progress.update()
def clear_iobuf(self, core_subsets, n_cores):
self._progress = ProgressBar(
n_cores, "clearing IOBUF from the machine")
for core_subset in core_subsets:
for processor_id in core_subset.processor_ids:
self._send_request(
SCPClearIOBUFRequest(
core_subset.x, core_subset.y, processor_id,
SDP_PORTS.RUNNING_COMMAND_SDP_PORT.value),
callback=self.receive_response)
self._finish()
self._progress.end()
self.check_for_error()
def call(self, inputs):
# Get the inputs to pass as the arguments
arg_inputs = self._get_inputs(inputs)
# Convert the arguments using the inputs
args = [
arg.format(**arg_inputs) for arg in self._command_line_arguments
]
algorithm_progress_bar = ProgressBar(
1, "Running external algorithm {}".format(self._algorithm_id))
# Run the external command
child = subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
child.wait()
algorithm_progress_bar.update(1)
algorithm_progress_bar.end()
# Detect any errors
if child.returncode != 0:
stdout, stderr = child.communicate()
raise PacmanExternalAlgorithmFailedToCompleteException(
"Algorithm {} returned a non-zero error code {}\n"
" Inputs: {}\n"
" Output: {}\n"
" Error: {}\n".format(
self._algorithm_id, child.returncode,
inputs.keys(), stdout, stderr))
# Return the results processed into a dict
# Use None here as the results don't actually exist, and are expected
# to be obtained from a file, whose name is in inputs
return self._get_outputs(inputs, [None] * len(self._outputs))
def __call__(self, machine_graph, plan_n_timesteps, file_path):
"""
:param machine_graph: The graph to convert
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:param file_path: Where to write the JSON
"""
progress = ProgressBar(
machine_graph.n_vertices + 1, "Converting to JSON graph")
# write basic stuff
json_graph = dict()
# write vertices data
vertices_resources = dict()
json_graph["vertices_resources"] = vertices_resources
edges_resources = defaultdict()
json_graph["edges"] = edges_resources
vertex_by_id = dict()
partition_by_id = dict()
for vertex in progress.over(machine_graph.vertices, False):
self._convert_vertex(
vertex, vertex_by_id, vertices_resources, edges_resources,
machine_graph, plan_n_timesteps, partition_by_id)
with open(file_path, "w") as f:
json.dump(json_graph, f)
progress.update()
file_format_schemas.validate(json_graph, "machine_graph.json")
progress.end()
return file_path, vertex_by_id, partition_by_id
def __call__(self, application_graph, graph_mapper, machine_graph,
n_keys_map):
"""
:param application_graph: The application graph
:param graph_mapper: the mapping between graphs
:param machine_graph: the machine graph
:param n_keys_map: the mapping between edges and n keys
:return: routing information objects
"""
progress_bar = ProgressBar(
machine_graph.n_outgoing_edge_partitions * 3,
"Allocating routing keys")
# ensure groups are stable and correct
self._determine_groups(
machine_graph, graph_mapper, application_graph, n_keys_map,
progress_bar)
# define the key space
bit_field_space = BitField(32)
field_positions = set()
# locate however many types of constraints there are
seen_fields = deduce_types(machine_graph)
progress_bar.update(machine_graph.n_outgoing_edge_partitions)
if len(seen_fields) > 1:
self._adds_application_field_to_the_fields(seen_fields)
# handle the application space
self._create_application_space_in_the_bit_field_space(
bit_field_space, seen_fields, field_positions)
# assign fields to positions in the space
bit_field_space.assign_fields()
# get positions of the flexible fields:
self._assign_flexi_field_positions(
bit_field_space, seen_fields, field_positions)
# create routing_info_allocator
routing_info = RoutingInfo()
seen_mask_instances = 0
# extract keys and masks for each edge from the bitfield
for partition in machine_graph.outgoing_edge_partitions:
# get keys and masks
keys_and_masks, seen_mask_instances = \
self._extract_keys_and_masks_from_bit_field(
partition, bit_field_space, n_keys_map,
seen_mask_instances)
# update routing info for each edge in the partition
partition_info = PartitionRoutingInfo(keys_and_masks, partition)
routing_info.add_partition_info(partition_info)
# update the progress bar again
progress_bar.update()
progress_bar.end()
return routing_info, field_positions
def __call__(self, machine_graph, n_keys_map, routing_tables):
# check that this algorithm supports the constraints
check_algorithm_can_support_constraints(
constrained_vertices=machine_graph.outgoing_edge_partitions,
supported_constraints=[
FixedMaskConstraint,
FixedKeyAndMaskConstraint,
ContiguousKeyRangeContraint],
abstract_constraint_type=AbstractKeyAllocatorConstraint)
# verify that no edge has more than 1 of a constraint ,and that
# constraints are compatible
check_types_of_edge_constraint(machine_graph)
routing_infos = RoutingInfo()
# Get the edges grouped by those that require the same key
(fixed_keys, _shared_keys, fixed_masks, fixed_fields, flexi_fields,
continuous, noncontinuous) = \
get_edge_groups(machine_graph, EdgeTrafficType.MULTICAST)
if flexi_fields:
raise PacmanConfigurationException(
"MallocBasedRoutingInfoAllocator does not support FlexiField")
# Even non-continuous keys will be continuous
for group in noncontinuous:
continuous.add(group)
# Go through the groups and allocate keys
progress = ProgressBar(
machine_graph.n_outgoing_edge_partitions,
"Allocating routing keys")
# allocate the groups that have fixed keys
for group in progress.over(fixed_keys, False):
# Get any fixed keys and masks from the group and attempt to
# allocate them
fixed_mask = None
fixed_key_and_mask_constraint = locate_constraints_of_type(
group.constraints, FixedKeyAndMaskConstraint)[0]
# attempt to allocate them
self._allocate_fixed_keys_and_masks(
fixed_key_and_mask_constraint.keys_and_masks, fixed_mask)
# update the pacman data objects
self._update_routing_objects(
fixed_key_and_mask_constraint.keys_and_masks, routing_infos,
group)
continuous.remove(group)
for group in progress.over(fixed_masks, False):
# get mask and fields if need be
fixed_mask = locate_constraints_of_type(
group.constraints, FixedMaskConstraint)[0].mask
fields = None
if group in fixed_fields:
fields = locate_constraints_of_type(
group.constraints, FixedKeyFieldConstraint)[0].fields
fixed_fields.remove(group)
# try to allocate
keys_and_masks = self._allocate_keys_and_masks(
fixed_mask, fields, n_keys_map.n_keys_for_partition(group))
# update the pacman data objects
self._update_routing_objects(keys_and_masks, routing_infos, group)
continuous.remove(group)
for group in progress.over(fixed_fields, False):
fields = locate_constraints_of_type(
group.constraints, FixedKeyFieldConstraint)[0].fields
# try to allocate
keys_and_masks = self._allocate_keys_and_masks(
None, fields, n_keys_map.n_keys_for_partition(group))
# update the pacman data objects
self._update_routing_objects(keys_and_masks, routing_infos, group)
continuous.remove(group)
# Sort the rest of the groups, using the routing tables for guidance
# Group partitions by those which share routes in any table
partition_groups = OrderedDict()
routers = reversed(sorted(
routing_tables.get_routers(),
key=lambda item: len(routing_tables.get_entries_for_router(
item[0], item[1]))))
for x, y in routers:
# Find all partitions that share a route in this table
partitions_by_route = defaultdict(OrderedSet)
routing_table = routing_tables.get_entries_for_router(x, y)
for partition, entry in iteritems(routing_table):
if partition in continuous:
entry_hash = sum(
1 << i
for i in entry.link_ids)
entry_hash += sum(
1 << (i + 6)
for i in entry.processor_ids)
partitions_by_route[entry_hash].add(partition)
for entry_hash, partitions in iteritems(partitions_by_route):
found_groups = list()
for partition in partitions:
if partition in partition_groups:
found_groups.append(partition_groups[partition])
if not found_groups:
# If no group was found, create a new one
for partition in partitions:
partition_groups[partition] = partitions
elif len(found_groups) == 1:
# If a single other group was found, merge it
for partition in partitions:
found_groups[0].add(partition)
partition_groups[partition] = found_groups[0]
else:
# Merge the groups
new_group = partitions
for group in found_groups:
for partition in group:
new_group.add(partition)
for partition in new_group:
partition_groups[partition] = new_group
# Sort partitions by largest group
continuous = list(OrderedSet(
tuple(group) for group in itervalues(partition_groups)))
for group in reversed(sorted(continuous, key=len)):
for partition in progress.over(group, False):
keys_and_masks = self._allocate_keys_and_masks(
None, None, n_keys_map.n_keys_for_partition(partition))
# update the pacman data objects
self._update_routing_objects(
keys_and_masks, routing_infos, partition)
progress.end()
return routing_infos
def __call__(
self, graph, machine, plan_n_timesteps,
preallocated_resources=None):
"""
:param graph: The application_graph to partition
:type graph:\
:py:class:`pacman.model.graph.application.ApplicationGraph`
:param machine: The machine with respect to which to partition the\
application_graph
:type machine: :py:class:`spinn_machine.Machine`
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:return: \
A machine_graph of partitioned vertices and partitioned edges
:rtype:\
:py:class:`pacman.model.graph.machine.MachineGraph`
:raise pacman.exceptions.PacmanPartitionException: \
If something goes wrong with the partitioning
"""
ResourceTracker.check_constraints(graph.vertices)
utils.check_algorithm_can_support_constraints(
constrained_vertices=graph.vertices,
abstract_constraint_type=AbstractPartitionerConstraint,
supported_constraints=[MaxVertexAtomsConstraint,
SameAtomsAsVertexConstraint,
FixedVertexAtomsConstraint])
# Load the vertices and create the machine_graph to fill
machine_graph = MachineGraph(
label="partitioned graph for {}".format(graph.label))
graph_mapper = GraphMapper()
# sort out vertex's by placement constraints
vertices = sort_vertices_by_known_constraints(graph.vertices)
# Set up the progress
n_atoms = 0
for vertex in vertices:
n_atoms += vertex.n_atoms
progress = ProgressBar(n_atoms, "Partitioning graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps,
preallocated_resources=preallocated_resources)
# Group vertices that are supposed to be the same size
vertex_groups = get_same_size_vertex_groups(vertices)
# Partition one vertex at a time
for vertex in vertices:
# check that the vertex hasn't already been partitioned
machine_vertices = graph_mapper.get_machine_vertices(vertex)
# if not, partition
if machine_vertices is None:
self._partition_vertex(
vertex, plan_n_timesteps, machine_graph, graph_mapper,
resource_tracker, progress, vertex_groups)
progress.end()
generate_machine_edges(machine_graph, graph_mapper, graph)
return machine_graph, graph_mapper, resource_tracker.chips_used
def __call__(
self, routing_tables, transceiver, machine, app_id,
provenance_file_path, compress_only_when_needed=True,
compress_as_much_as_possible=False):
"""
:param routing_tables: the memory routing tables to be compressed
:param transceiver: the spinnman interface
:param machine: the SpiNNaker machine representation
:param app_id: the application ID used by the main application
:param provenance_file_path: the path to where to write the data
:return: flag stating routing compression and loading has been done
"""
# pylint: disable=too-many-arguments
# build progress bar
progress = ProgressBar(
len(routing_tables.routing_tables) + 2,
"Running routing table compression on chip")
compressor_app_id = transceiver.app_id_tracker.get_new_id()
# figure size of SDRAM needed for each chip for storing the routing
# table
for routing_table in progress.over(routing_tables, False):
self._load_routing_table(
routing_table, transceiver, app_id, compressor_app_id,
compress_only_when_needed, compress_as_much_as_possible)
# load the router compressor executable
executable_targets = self._load_executables(
routing_tables, compressor_app_id, transceiver, machine)
# update progress bar
progress.update()
# Wait for the executable to finish
succeeded = False
try:
transceiver.wait_for_cores_to_be_in_state(
executable_targets.all_core_subsets, compressor_app_id,
[CPUState.FINISHED])
succeeded = True
finally:
# get the debug data
if not succeeded:
self._handle_failure(
executable_targets, transceiver, provenance_file_path,
compressor_app_id)
# Check if any cores have not completed successfully
self._check_for_success(
executable_targets, transceiver,
provenance_file_path, compressor_app_id)
# update progress bar
progress.update()
# stop anything that's associated with the compressor binary
transceiver.stop_application(compressor_app_id)
transceiver.app_id_tracker.free_id(compressor_app_id)
# update the progress bar
progress.end()