def __call__(self,
live_packet_gatherer_parameters,
machine,
machine_graph,
application_graph=None,
graph_mapper=None):
""" Add LPG vertices on Ethernet connected chips as required.
:param live_packet_gatherer_parameters:\
the Live Packet Gatherer parameters requested by the script
:param machine: the SpiNNaker machine as discovered
:param application_graph: the application graph
:param machine_graph: the machine graph
:return: mapping between LPG parameters and LPG vertex
"""
# pylint: disable=too-many-arguments
# create progress bar
progress = ProgressBar(machine.ethernet_connected_chips,
string_describing_what_being_progressed=(
"Adding Live Packet Gatherers to Graph"))
# Keep track of the vertices added by parameters and board address
lpg_params_to_vertices = defaultdict(dict)
# for every Ethernet connected chip, add the gatherers required
for chip in progress.over(machine.ethernet_connected_chips):
for params in live_packet_gatherer_parameters:
if (params.board_address is None
or params.board_address == chip.ip_address):
lpg_params_to_vertices[params][chip.x, chip.y] = \
self._add_lpg_vertex(application_graph, graph_mapper,
machine_graph, chip, params)
return lpg_params_to_vertices
def _run_for_core_subsets(self, core_subsets, transceiver,
provenance_file_path):
progress = ProgressBar(len(core_subsets),
"Extracting IOBUF from the machine")
error_entries = list()
warn_entries = list()
# extract iobuf
io_buffers = list(transceiver.get_iobuf(core_subsets))
# write iobuf
for iobuf in io_buffers:
file_name = os.path.join(
provenance_file_path,
"{}_{}_{}.txt".format(iobuf.x, iobuf.y, iobuf.p))
# set mode of the file based off if the file already exists
mode = "w"
if os.path.exists(file_name):
mode = "a"
# write iobuf to file.
with open(file_name, mode) as writer:
if mode == "a":
writer.write("\n\n")
writer.write(iobuf.iobuf)
# check iobuf for errors
for io_buffer in progress.over(io_buffers):
self._check_iobuf_for_error(io_buffer, error_entries, warn_entries)
return error_entries, warn_entries
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.IDLE)
if error_cores or watchdog_cores or idle_cores:
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(error_cores.values(),
watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
self._update_provenance(txrx, total_processors, processors_completed,
all_core_subsets, app_id, progress)
progress.end()
def __call__(self, router_tables, app_id, transceiver, machine):
"""
:param ~.MulticastRoutingTables router_tables:
:param int app_id:
:param ~.Transceiver transceiver:
:param ~.Machine machine:
"""
routing_tables = list(router_tables.routing_tables)
progress = ProgressBar(routing_tables, "Preparing Routing Tables")
# Clear the routing table for each router that needs to be set up
# and set up the diagnostics
for router_table in progress.over(routing_tables):
if not machine.get_chip_at(router_table.x, router_table.y).virtual:
transceiver.clear_multicast_routes(
router_table.x, router_table.y)
transceiver.clear_router_diagnostic_counters(
router_table.x, router_table.y)
# set the router diagnostic for user 3 to catch local default
# routed packets. This can only occur when the source router
# has no router entry, and therefore should be detected a bad
# dropped packet.
self._set_router_diagnostic_filters(
router_table.x, router_table.y, transceiver)
def __call__(self, machine, extra_monitor_cores, placements):
"""
:param ~spinn_machine.Machine machine:
:param dict(tuple(int,int),ExtraMonitorSupportMachineVertex) \
extra_monitor_cores:
:param ~pacman.model.placements.Placements placements:
:rtype: tuple(MulticastRoutingTables,
dict(tuple(int,int),int), dict(tuple(int,int),int))
"""
# pylint: disable=attribute-defined-outside-init
self._machine = machine
self._placements = placements
self._monitors = extra_monitor_cores
self._routing_tables = MulticastRoutingTables()
self._key_to_destination_map = dict()
self._time_out_keys_by_board = dict()
# create progress bar
progress = ProgressBar(
machine.ethernet_connected_chips,
"Generating routing tables for data in system processes")
for ethernet_chip in progress.over(machine.ethernet_connected_chips):
tree = self._generate_routing_tree(ethernet_chip)
self._add_routing_entries(ethernet_chip, tree)
return (self._routing_tables, self._key_to_destination_map,
self._time_out_keys_by_board)
def get_spikes(self, label, buffer_manager, region,
placements, graph_mapper, application_vertex,
base_key_function, machine_time_step):
# pylint: disable=too-many-arguments
results = list()
missing = []
ms_per_tick = machine_time_step / 1000.0
vertices = graph_mapper.get_machine_vertices(application_vertex)
progress = ProgressBar(vertices,
"Getting spikes for {}".format(label))
for vertex in progress.over(vertices):
placement = placements.get_placement_of_vertex(vertex)
vertex_slice = graph_mapper.get_slice(vertex)
# Read the spikes
raw_spike_data, data_missing = \
buffer_manager.get_data_by_placement(placement, region)
if data_missing:
missing.append(placement)
self._process_spike_data(
vertex_slice, raw_spike_data, ms_per_tick,
base_key_function(vertex), results)
if missing:
missing_str = recording_utils.make_missing_string(missing)
logger.warning(
"Population {} is missing spike data in region {} from the"
" following cores: {}", label, region, missing_str)
if not results:
return numpy.empty(shape=(0, 2))
result = numpy.vstack(results)
return result[numpy.lexsort((result[:, 1], result[:, 0]))]
def placement_report_without_application_graph_by_core(report_folder, hostname,
placements, machine):
""" Generate report on the placement of vertices onto cores by core.
:param str report_folder: the folder to which the reports are being written
:param str hostname: the machine's hostname to which the placer worked on
:param ~spinn_machine.Machine machine: the SpiNNaker machine object
:param Placements placements: the placements objects built by the placer.
"""
# File 2: Placement by core.
# Cycle through all chips and by all cores within each chip.
# For each core, display what is held on it.
file_name = os.path.join(report_folder, _PLACEMENT_CORE_SIMPLE_FILENAME)
time_date_string = time.strftime("%c")
try:
with open(file_name, "w") as f:
progress = ProgressBar(machine.chips,
"Generating placement by core report")
f.write(" Placement Information by Core\n")
f.write(" =============================\n\n")
f.write("Generated: {}".format(time_date_string))
f.write(" for target machine '{}'".format(hostname))
f.write("\n\n")
for chip in progress.over(machine.chips):
_write_one_chip_machine_placement(f, chip, placements)
except IOError:
logger.exception(
"Generate_placement_reports: Can't open file {} for "
"writing.", file_name)
def __call__(
self, transceiver, tags=None, iptags=None, reverse_iptags=None):
"""
:param tags: the tags object which contains IP and reverse IP tags.
could be none if these are being given in separate lists
:param iptags: a list of IP tags, given when tags is none
:param reverse_iptags: a list of reverse IP tags when tags is none.
:param transceiver: the transceiver object
"""
# clear all the tags from the Ethernet connection, as nothing should
# be allowed to use it (no two apps should use the same Ethernet
# connection at the same time)
progress = ProgressBar(MAX_TAG_ID, "Clearing tags")
for tag_id in progress.over(range(MAX_TAG_ID)):
transceiver.clear_ip_tag(tag_id)
# Use tags object to supply tag info if it is supplied
if tags is not None:
iptags = list(tags.ip_tags)
reverse_iptags = list(tags.reverse_ip_tags)
# Load the IP tags and the Reverse IP tags
progress = ProgressBar(
len(iptags) + len(reverse_iptags), "Loading Tags")
self.load_iptags(iptags, transceiver, progress)
self.load_reverse_iptags(reverse_iptags, transceiver, progress)
progress.end()
def __call__(self, report_folder, connection_holder, dsg_targets):
""" Convert synaptic matrix for every application edge.
"""
# Update the print options to display everything
print_opts = numpy.get_printoptions()
numpy.set_printoptions(threshold=numpy.nan)
if dsg_targets is None:
raise SynapticConfigurationException(
"dsg_targets should not be none, used as a check for "
"connection holder data to be generated")
# generate folder for synaptic reports
top_level_folder = os.path.join(report_folder, _DIRNAME)
if not os.path.exists(top_level_folder):
os.mkdir(top_level_folder)
# create progress bar
progress = ProgressBar(connection_holder.keys(),
"Generating synaptic matrix reports")
# for each application edge, write matrix in new file
for edge, _ in progress.over(connection_holder.keys()):
# only write matrix's for edges which have matrix's
if isinstance(edge, ProjectionApplicationEdge):
# figure new file name
file_name = os.path.join(top_level_folder,
_TMPL_FILENAME.format(edge.label))
self._write_file(file_name, connection_holder, edge)
# Reset the print options
numpy.set_printoptions(**print_opts)
def __call__(
self, live_packet_gatherer_parameters, machine, machine_graph,
application_graph=None, graph_mapper=None):
""" Add LPG vertices on Ethernet connected chips as required.
:param live_packet_gatherer_parameters:\
the Live Packet Gatherer parameters requested by the script
:param machine: the SpiNNaker machine as discovered
:param application_graph: the application graph
:param machine_graph: the machine graph
:return: mapping between LPG parameters and LPG vertex
"""
# pylint: disable=too-many-arguments
# create progress bar
progress = ProgressBar(
machine.ethernet_connected_chips,
string_describing_what_being_progressed=(
"Adding Live Packet Gatherers to Graph"))
# Keep track of the vertices added by parameters and board address
lpg_params_to_vertices = defaultdict(dict)
# for every Ethernet connected chip, add the gatherers required
for chip in progress.over(machine.ethernet_connected_chips):
for params in live_packet_gatherer_parameters:
if (params.board_address is None or
params.board_address == chip.ip_address):
lpg_params_to_vertices[params][chip.x, chip.y] = \
self._add_lpg_vertex(application_graph, graph_mapper,
machine_graph, chip, params)
return lpg_params_to_vertices
def __call__(
self, transceiver, placements, provenance_file_path,
run_time_ms, machine_time_step):
"""
:param transceiver: the SpiNNMan interface object
:param placements: The placements of the vertices
:param has_ran: token that states that the simulation has ran
:param provenance_file_path: The location to store the profile data
:param run_time_ms: runtime in ms
:param machine_time_step: machine time step in ms
"""
# pylint: disable=too-many-arguments
machine_time_step_ms = machine_time_step // 1000
progress = ProgressBar(
placements.n_placements, "Getting profile data")
# retrieve provenance data from any cores that provide data
for placement in progress.over(placements.placements):
if isinstance(placement.vertex, AbstractHasProfileData):
# get data
profile_data = placement.vertex.get_profile_data(
transceiver, placement)
if profile_data.tags:
self._write(placement, profile_data, run_time_ms,
machine_time_step_ms, provenance_file_path)
def __call__(
self, transceiver, placements, provenance_file_path,
machine_time_step):
"""
:param ~.Transceiver transceiver:
:param ~.Placements placements:
:param str provenance_file_path:
:param int machine_time_step:
"""
# pylint: disable=too-many-arguments
machine_time_step_ms = (
float(machine_time_step) / MICRO_TO_MILLISECOND_CONVERSION)
progress = ProgressBar(
placements.n_placements, "Getting profile data")
# retrieve provenance data from any cores that provide data
for placement in progress.over(placements.placements):
if isinstance(placement.vertex, AbstractHasProfileData):
# get data
profile_data = placement.vertex.get_profile_data(
transceiver, placement)
if profile_data.tags:
self._write(placement, profile_data, machine_time_step_ms,
provenance_file_path)
def __call__(self, machine_graph, graph_mapper):
"""
:param machine_graph: the machine_graph whose edges are to be filtered
:param graph_mapper: the graph mapper between graphs
:return: a new graph mapper and machine graph
"""
new_machine_graph = MachineGraph(label=machine_graph.label)
new_graph_mapper = GraphMapper()
# create progress bar
progress = ProgressBar(
machine_graph.n_vertices +
machine_graph.n_outgoing_edge_partitions, "Filtering edges")
# add the vertices directly, as they wont be pruned.
for vertex in progress.over(machine_graph.vertices, False):
self._add_vertex_to_new_graph(vertex, graph_mapper,
new_machine_graph, new_graph_mapper)
# start checking edges to decide which ones need pruning....
for partition in progress.over(machine_graph.outgoing_edge_partitions):
for edge in partition.edges:
if self._is_filterable(edge, graph_mapper):
logger.debug("this edge was pruned %s", edge)
continue
logger.debug("this edge was not pruned %s", edge)
self._add_edge_to_new_graph(edge, partition, graph_mapper,
new_machine_graph,
new_graph_mapper)
# returned the pruned graph and graph_mapper
return new_machine_graph, new_graph_mapper
def __call__(self, machine_graph, machine, plan_n_timesteps):
""" Place each vertex in a machine graph on a core in the machine.
:param machine_graph: The machine_graph to place
:type machine_graph:\
:py:class:`pacman.model.graphs.machine.MachineGraph`
:param machine: A SpiNNaker machine object.
:type machine: :py:class:`spinn_machine.Machine`
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:return placements: Placements of vertices on the machine
:rtype :py:class:`pacman.model.placements.Placements`
"""
# check that the algorithm can handle the constraints
ResourceTracker.check_constraints(machine_graph.vertices)
placements = Placements()
vertices = sort_vertices_by_known_constraints(machine_graph.vertices)
# Iterate over vertices and generate placements
progress = ProgressBar(machine_graph.n_vertices,
"Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_random_chips(machine))
vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
vertices_placed = set()
for vertex in progress.over(vertices):
if vertex not in vertices_placed:
vertices_placed.update(self._place_vertex(
vertex, resource_tracker, machine, placements,
vertices_on_same_chip))
return placements
def __call__(self, router_tables):
tables = MulticastRoutingTables()
previous_masks = dict()
progress = ProgressBar(
len(router_tables.routing_tables) * 2,
"Compressing Routing Tables")
# Create all masks without holes
allowed_masks = [_32_BITS - ((2**i) - 1) for i in range(33)]
# Check that none of the masks have "holes" e.g. 0xFFFF0FFF has a hole
for router_table in router_tables.routing_tables:
for entry in router_table.multicast_routing_entries:
if entry.mask not in allowed_masks:
raise PacmanRoutingException(
"Only masks without holes are allowed in tables for"
" MallocBasedRouteMerger (disallowed mask={})".format(
hex(entry.mask)))
for router_table in progress.over(router_tables.routing_tables):
new_table = self._merge_routes(router_table, previous_masks)
tables.add_routing_table(new_table)
n_entries = len([
entry for entry in new_table.multicast_routing_entries
if not entry.defaultable
])
print("Reduced from {} to {}".format(
len(router_table.multicast_routing_entries), n_entries))
if n_entries > 1023:
raise PacmanRoutingException(
"Cannot make table small enough: {} entries".format(
n_entries))
return tables
def __call__(self, router_tables):
tables = MulticastRoutingTables()
previous_masks = dict()
progress = ProgressBar(
len(router_tables.routing_tables) * 2,
"Compressing Routing Tables")
# Create all masks without holes
allowed_masks = [_32_BITS - ((2 ** i) - 1) for i in range(33)]
# Check that none of the masks have "holes" e.g. 0xFFFF0FFF has a hole
for router_table in router_tables.routing_tables:
for entry in router_table.multicast_routing_entries:
if entry.mask not in allowed_masks:
raise PacmanRoutingException(
"Only masks without holes are allowed in tables for"
" BasicRouteMerger (disallowed mask={})".format(
hex(entry.mask)))
for router_table in progress.over(router_tables.routing_tables):
new_table = self._merge_routes(router_table, previous_masks)
tables.add_routing_table(new_table)
n_entries = len([
entry for entry in new_table.multicast_routing_entries
if not entry.defaultable])
# print("Reduced from {} to {}".format(
# len(router_table.multicast_routing_entries), n_entries))
if n_entries > 1023:
raise PacmanRoutingException(
"Cannot make table small enough: {} entries".format(
n_entries))
return tables
def __call__(self, machine_graph, machine, plan_n_timesteps):
""" Place each vertex in a machine graph on a core in the machine.
:param MachineGraph machine_graph: The machine_graph to place
:param ~spinn_machine.Machine machine: A SpiNNaker machine object.
:param int plan_n_timesteps: number of timesteps to plan for
:return placements: Placements of vertices on the machine
:rtype: Placements
"""
# check that the algorithm can handle the constraints
ResourceTracker.check_constraints(machine_graph.vertices)
placements = Placements()
vertices = sort_vertices_by_known_constraints(machine_graph.vertices)
# Iterate over vertices and generate placements
progress = ProgressBar(machine_graph.n_vertices,
"Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_random_chips(machine))
vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
vertices_placed = set()
for vertex in progress.over(vertices):
if vertex not in vertices_placed:
vertices_placed.update(
self._place_vertex(vertex, resource_tracker, machine,
placements, vertices_on_same_chip))
return placements
def __call__(self, router_tables, target_length=None):
# build storage
compressed_pacman_router_tables = MulticastRoutingTables()
# create progress bar
progress = ProgressBar(
router_tables.routing_tables, "Compressing routing Tables")
# compress each router
for router_table in progress.over(router_tables.routing_tables):
# convert to rig format
entries = self._convert_to_mundy_format(router_table)
# compress the router entries
compressed_router_table_entries = \
rigs_compressor.minimise(entries, target_length)
# convert back to pacman model
compressed_pacman_table = self._convert_to_pacman_router_table(
compressed_router_table_entries, router_table.x,
router_table.y)
# add to new compressed routing tables
compressed_pacman_router_tables.add_routing_table(
compressed_pacman_table)
# return
return compressed_pacman_router_tables
def partitioner_report(report_folder, hostname, graph):
""" Generate report on the placement of vertices onto cores.
:param str report_folder: the folder to which the reports are being written
:param str hostname: the machine's hostname to which the placer worked on
:param ApplicationGraph graph: the app graph
"""
# Cycle through all vertices, and for each cycle through its vertices.
# For each vertex, describe its core mapping.
file_name = os.path.join(report_folder, _PARTITIONING_FILENAME)
time_date_string = time.strftime("%c")
try:
with open(file_name, "w") as f:
progress = ProgressBar(graph.n_vertices,
"Generating partitioner report")
f.write(" Partitioning Information by Vertex\n")
f.write(" ===============================\n\n")
f.write("Generated: {} for target machine '{}'\n\n".format(
time_date_string, hostname))
for vertex in progress.over(graph.vertices):
_write_one_vertex_partition(f, vertex)
except IOError:
logger.exception(
"Generate_placement_reports: Can't open file {} for"
" writing.", file_name)
def __call__(self, report_default_directory, dsg_targets, transceiver):
""" Creates a report that states where in SDRAM each region is \
(read from machine)
:param report_default_directory: the folder where reports are written
:param dsg_targets: the map between placement and file writer
:param transceiver: the spinnMan instance
:rtype: None
"""
directory_name = os.path.join(
report_default_directory, MEM_MAP_SUBDIR_NAME)
if not os.path.exists(directory_name):
os.makedirs(directory_name)
progress = ProgressBar(dsg_targets, "Writing memory map reports")
for (x, y, p) in progress.over(dsg_targets):
file_name = os.path.join(
directory_name, MEM_MAP_FILENAME.format(x, y, p))
try:
with open(file_name, "w") as f:
self._describe_mem_map(f, transceiver, x, y, p)
except IOError:
logger.exception("Generate_placement_reports: Can't open file"
" {} for writing.", file_name)
def get_spikes(self, label, buffer_manager, region, placements,
graph_mapper, application_vertex, base_key_function,
machine_time_step):
# pylint: disable=too-many-arguments
results = list()
missing = []
ms_per_tick = machine_time_step / 1000.0
vertices = graph_mapper.get_machine_vertices(application_vertex)
progress = ProgressBar(vertices, "Getting spikes for {}".format(label))
for vertex in progress.over(vertices):
placement = placements.get_placement_of_vertex(vertex)
vertex_slice = graph_mapper.get_slice(vertex)
# Read the spikes
raw_spike_data, data_missing = \
buffer_manager.get_data_by_placement(placement, region)
if data_missing:
missing.append(placement)
self._process_spike_data(vertex_slice, raw_spike_data, ms_per_tick,
base_key_function(vertex), results)
if missing:
missing_str = recording_utils.make_missing_string(missing)
logger.warning(
"Population {} is missing spike data in region {} from the"
" following cores: {}", label, region, missing_str)
if not results:
return numpy.empty(shape=(0, 2))
result = numpy.vstack(results)
return result[numpy.lexsort((result[:, 1], result[:, 0]))]
def _fill_in_connection_data(transceiver, expanded_pop_vertices, placements):
""" Once expander has run, fill in the connection data
:rtype: None
"""
ctl = globals_variables.get_simulator()
use_extra_monitors = False
progress_bar = ProgressBar(
len(expanded_pop_vertices),
"reading back synaptic data required for connection holders")
for vertex in progress_bar.over(expanded_pop_vertices):
conn_holders = vertex.get_connection_holders()
for (app_edge,
synapse_info), conn_holder_list in iteritems(conn_holders):
# Only do this if this synapse_info has been generated
# on the machine using the expander
if synapse_info.may_generate_on_machine():
machine_edges = app_edge.machine_edges
for machine_edge in machine_edges:
placement = placements.get_placement_of_vertex(
machine_edge.post_vertex)
conns = vertex.get_connections_from_machine(
transceiver, placement, machine_edge,
ctl.routing_infos, synapse_info, ctl.machine_time_step,
use_extra_monitors)
for conn_holder in conn_holder_list:
conn_holder.add_connections(conns)
for conn_holder in conn_holder_list:
conn_holder.finish()
def _ner_route(machine_graph, machine, placements, vector_to_nodes):
""" Performs routing using rig algorithm
:param MachineGraph machine_graph:
:param ~spinn_machine.Machine machine:
:param Placements placements:
:return:
:rtype: MulticastRoutingTableByPartition
"""
routing_tables = MulticastRoutingTableByPartition()
progress_bar = ProgressBar(len(machine_graph.vertices), "Routing")
for source_vertex in progress_bar.over(machine_graph.vertices):
# handle the vertex edges
for partition in machine_graph.\
get_multicast_edge_partitions_starting_at_vertex(
source_vertex):
post_vertexes = list(e.post_vertex for e in partition.edges)
routing_tree = _do_route(source_vertex, post_vertexes, machine,
placements, vector_to_nodes)
incoming_processor = placements.get_placement_of_vertex(
partition.pre_vertex).p
_convert_a_route(routing_tables, partition, incoming_processor,
None, routing_tree)
progress_bar.end()
return routing_tables
def __call__(self, machine_graph, machine, plan_n_timesteps):
""" Place a machine_graph so that each vertex is placed on a core
:param machine_graph: The machine_graph to place
:type machine_graph:\
:py:class:`pacman.model.graphs.machine.MachineGraph`
:param machine:\
The machine with respect to which to partition the application\
graph
:type machine: :py:class:`spinn_machine.Machine`
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:return: A set of placements
:rtype: :py:class:`pacman.model.placements.Placements`
:raise pacman.exceptions.PacmanPlaceException: \
If something goes wrong with the placement
"""
# check that the algorithm can handle the constraints
ResourceTracker.check_constraints(machine_graph.vertices)
placements = Placements()
vertices = sort_vertices_by_known_constraints(machine_graph.vertices)
# Iterate over vertices and generate placements
progress = ProgressBar(vertices, "Placing graph vertices")
resource_tracker = ResourceTracker(machine, plan_n_timesteps)
for vertex in progress.over(vertices):
# Create and store a new placement anywhere on the board
(x, y, p, _, _) = resource_tracker.allocate_constrained_resources(
vertex.resources_required, vertex.constraints, None)
placement = Placement(vertex, x, y, p)
placements.add_placement(placement)
return placements
def __call__(self, machine_graph, placements, n_keys_map):
"""
:param MachineGraph machine_graph:
The graph to allocate the routing info for
:param Placements placements: The placements of the vertices
:param AbstractMachinePartitionNKeysMap n_keys_map:
A map between the edges and the number of keys required by the
edges
:return: The routing information
:rtype: tuple(RoutingInfo, AbstractMulticastRoutingTable)
:raise PacmanRouteInfoAllocationException:
If something goes wrong with the allocation
"""
# check that this algorithm supports the constraints put onto the
# partitions
check_algorithm_can_support_constraints(
constrained_vertices=machine_graph.partitions,
supported_constraints=[],
abstract_constraint_type=AbstractKeyAllocatorConstraint)
# take each edge and create keys from its placement
progress = ProgressBar(machine_graph.n_outgoing_edge_partitions,
"Allocating routing keys")
routing_infos = RoutingInfo()
routing_tables = MulticastRoutingTables()
for partition in progress.over(machine_graph.outgoing_edge_partitions):
for edge in partition.edges:
routing_infos.add_partition_info(
self._allocate_partition_route(edge, placements,
machine_graph, n_keys_map))
return routing_infos, routing_tables
def __call__(self, machine_graph, machine, plan_n_timesteps):
"""
:param MachineGraph machine_graph: The machine_graph to place
:param ~spinn_machine.Machine machine:
The machine with respect to which to partition the application
graph
:param int plan_n_timesteps: number of timesteps to plan for
:return: A set of placements
:rtype: Placements
:raise PacmanPlaceException:
If something goes wrong with the placement
"""
# check that the algorithm can handle the constraints
self._check_constraints(machine_graph.vertices)
placements = Placements()
vertices = sort_vertices_by_known_constraints(machine_graph.vertices)
# Iterate over vertices and generate placements
progress = ProgressBar(machine_graph.n_vertices,
"Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_radial_chips(machine))
vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
all_vertices_placed = set()
for vertex in progress.over(vertices):
if vertex not in all_vertices_placed:
vertices_placed = self._place_vertex(vertex, resource_tracker,
machine, placements,
vertices_on_same_chip,
machine_graph)
all_vertices_placed.update(vertices_placed)
return placements
def __call__(self, app_graph):
""" Add a splitter to every vertex that doesn't already have one.
:param ApplicationGraph app_graph: app graph
:raises PacmanConfigurationException: If a bad configuration is set
"""
progress_bar = ProgressBar(
string_describing_what_being_progressed=self.PROGRESS_BAR_NAME,
total_number_of_things_to_do=len(app_graph.vertices))
for app_vertex in progress_bar.over(app_graph.vertices):
if app_vertex.splitter is None:
if isinstance(app_vertex, AbstractPopulationVertex):
self.abstract_pop_heuristic(app_vertex)
elif isinstance(app_vertex, ApplicationSpiNNakerLinkVertex):
self.external_spinnaker_link_heuristic(app_vertex)
elif isinstance(app_vertex, ApplicationFPGAVertex):
self.external_fpga_link_heuristic(app_vertex)
elif isinstance(app_vertex, SpikeSourceArrayVertex):
self.spike_source_array_heuristic(app_vertex)
elif isinstance(app_vertex, SpikeSourcePoissonVertex):
self.spike_source_poisson_heuristic(app_vertex)
else: # go to basic selector. it might know what to do
self.vertex_selector(app_vertex)
if isinstance(app_vertex, AbstractAcceptsIncomingSynapses):
app_vertex.verify_splitter(app_vertex.splitter)
def __call__(self,
placements,
hostname,
report_default_directory,
write_text_specs,
machine,
data_n_timesteps,
placement_order=None):
"""
:param ~.Placements placements:
:param str hostname:
:param str report_default_directory:
:param bool write_text_specs:
:param ~.Machine machine:
:param int data_n_timesteps:
:param list(~.Placement) placement_order:
:rtype: tuple(DataSpecificationTargets, dict(tuple(int,int,int), int))
:raises ConfigurationException:
"""
# pylint: disable=too-many-arguments, too-many-locals
# pylint: disable=attribute-defined-outside-init
self._machine = machine
self._hostname = hostname
self._report_dir = report_default_directory
self._write_text = write_text_specs
# iterate though vertices and call generate_data_spec for each
# vertex
targets = DataSpecificationTargets(machine, self._report_dir)
if placement_order is None:
placement_order = placements.placements
progress = ProgressBar(placements.n_placements,
"Generating data specifications")
vertices_to_reset = list()
for placement in progress.over(placement_order):
# Try to generate the data spec for the placement
vertex = placement.vertex
generated = self.__generate_data_spec_for_vertices(
placement, vertex, targets, data_n_timesteps)
if generated and isinstance(vertex,
AbstractRewritesDataSpecification):
vertices_to_reset.append(vertex)
# If the spec wasn't generated directly, and there is an
# application vertex, try with that
if not generated and vertex.app_vertex is not None:
generated = self.__generate_data_spec_for_vertices(
placement, vertex.app_vertex, targets, data_n_timesteps)
if generated and isinstance(vertex.app_vertex,
AbstractRewritesDataSpecification):
vertices_to_reset.append(vertex.app_vertex)
# Ensure that the vertices know their regions have been reloaded
for vertex in vertices_to_reset:
vertex.mark_regions_reloaded()
return targets, self._region_sizes
def range_compressor(router_tables, accept_overflow=True):
"""
:param MulticastRoutingTables router_tables:
:param bool accept_overflow:
A flag which should only be used in testing to stop raising an
exception if result is too big
:rtype: MulticastRoutingTables
"""
if accept_overflow:
message = "Precompressing tables using Range Compressor"
else:
message = "Compressing tables using Range Compressor"
progress = ProgressBar(len(router_tables.routing_tables), message)
compressor = RangeCompressor()
compressed_tables = MulticastRoutingTables()
for table in progress.over(router_tables.routing_tables):
new_table = compressor.compress_table(table)
if (new_table.number_of_entries > Machine.ROUTER_ENTRIES
and not accept_overflow):
raise MinimisationFailedError(
f"The routing table {table.x} {table.y} with "
f"{table.number_of_entries} entries after compression "
f"still has {new_table.number_of_entries} so will not fit")
compressed_tables.add_routing_table(new_table)
logger.info(f"Ranged compressor resulted with the largest table of size "
f"{compressed_tables.max_number_of_entries}")
return compressed_tables
def __call__(self, machine_graph, machine, plan_n_timesteps):
"""
:param MachineGraph machine_graph: The machine_graph to place
:param ~spinn_machine.Machine machine: A SpiNNaker machine object.
:param int plan_n_timesteps: number of timesteps to plan for
:return: Placements of vertices on the machine
:rtype: Placements
"""
# check that the algorithm can handle the constraints
self._check_constraints(
machine_graph.vertices,
additional_placement_constraints={SameChipAsConstraint})
# in order to test isomorphism include:
# placements_copy = Placements()
placements = Placements()
vertices = sort_vertices_by_known_constraints(machine_graph.vertices)
progress = ProgressBar(
machine_graph.n_vertices, "Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_hilbert_chips(machine))
# get vertices which must be placed on the same chip
vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph)
# iterate over vertices and generate placements
all_vertices_placed = set()
for vertex in progress.over(vertices):
if vertex not in all_vertices_placed:
vertices_placed = self._place_vertex(
vertex, resource_tracker, machine,
placements, vertices_on_same_chip)
all_vertices_placed.update(vertices_placed)
return placements
def __call__(self, report_default_directory, dsg_targets, transceiver):
""" Creates a report that states where in SDRAM each region is \
(read from machine)
:param report_default_directory: the folder where reports are written
:param dsg_targets: the map between placement and file writer
:param transceiver: the spinnMan instance
:rtype: None
"""
directory_name = os.path.join(report_default_directory,
MEM_MAP_SUBDIR_NAME)
if not os.path.exists(directory_name):
os.makedirs(directory_name)
progress = ProgressBar(dsg_targets, "Writing memory map reports")
for (x, y, p) in progress.over(dsg_targets):
file_name = os.path.join(directory_name,
MEM_MAP_FILENAME.format(x, y, p))
try:
with open(file_name, "w") as f:
self._describe_mem_map(f, transceiver, x, y, p)
except IOError:
logger.exception(
"Generate_placement_reports: Can't open file"
" {} for writing.", file_name)
def compress(self, register):
""" Apply the on-machine compression algorithm.
:param int register: number of user register to check
"""
# pylint: disable=too-many-arguments
# build progress bar
progress_bar = ProgressBar(
len(self._routing_tables.routing_tables) * 2,
self._progresses_text)
self._compressor_app_id = self._transceiver.app_id_tracker.get_new_id()
# figure size of SDRAM needed for each chip for storing the routing
# table
for routing_table in progress_bar.over(self._routing_tables, False):
self._load_routing_table(routing_table)
# load the router compressor executable
executable_targets = self._load_executables()
executable_finder = ExecutableFinder(binary_search_paths=[])
read_algorithm_iobuf = True
run_system_application(
executable_targets, self._compressor_app_id, self._transceiver,
self._provenance_file_path, executable_finder,
read_algorithm_iobuf,
functools.partial(
self._check_for_success,
register=register),
[CPUState.FINISHED], False, "compressor_on_{}_{}_{}.txt",
[self._binary_path], progress_bar)
def __call__(
self, live_packet_gatherer_parameters, placements,
live_packet_gatherers_to_vertex_mapping, machine,
machine_graph, application_graph=None, graph_mapper=None):
"""
:param live_packet_gatherer_parameters: the set of parameters
:param placements: the placements object
:param live_packet_gatherers_to_vertex_mapping:\
the mapping of LPG parameters and the machine vertices associated\
with it
:param machine: the SpiNNaker machine
:param machine_graph: the machine graph
:param application_graph: the app graph
:param graph_mapper: the graph mapper between app and machine graph
:rtype: None
"""
# pylint: disable=too-many-arguments
progress = ProgressBar(
live_packet_gatherer_parameters,
string_describing_what_being_progressed=(
"Adding edges to the machine graph between the vertices to "
"which live output has been requested and its local Live "
"Packet Gatherer"))
for lpg_params in progress.over(live_packet_gatherer_parameters):
# locate vertices needed to be connected to a LPG with these params
for vertex in live_packet_gatherer_parameters[lpg_params]:
self._connect_lpg_vertex(
application_graph, graph_mapper, machine,
placements, machine_graph, vertex,
live_packet_gatherers_to_vertex_mapping, lpg_params)
def _read_back_bit_fields(self, app_graph, transceiver, placements,
default_report_folder, bit_field_report_name):
""" report of the bitfields that were generated
:param app_graph: app graph
:param transceiver: the SPiNNMan instance
:param placements: The placements
:param default_report_folder:the file path for where reports are.
:param bit_field_report_name: the name of the file
:rtype: None
"""
# generate file
progress = ProgressBar(len(app_graph.vertices),
"reading back bitfields from chip")
file_path = os.path.join(default_report_folder, bit_field_report_name)
with open(file_path, "w") as output:
# read in for each app vertex that would have a bitfield
for app_vertex in progress.over(app_graph.vertices):
# get machine verts
for machine_vertex in app_vertex.machine_vertices:
if isinstance(machine_vertex,
AbstractSupportsBitFieldGeneration):
self.__read_back_single_core_data(
machine_vertex, placements, transceiver, output)
def generate_initial_connectivity(sigma,
p,
msg,
N_layer=256,
n=16,
s_max=16,
g_max=.2,
delay=1.):
# print "|", 256 // 4 * "-", "|"
# print "|",
connections = []
s = np.zeros(N_layer)
pbar = ProgressBar(total_number_of_things_to_do=N_layer,
string_describing_what_being_progressed=msg)
for postsynaptic_neuron_index in range(N_layer):
# if postsynaptic_neuron_index % 8 == 0:
# print "=",
pbar.update()
post = (postsynaptic_neuron_index // n, postsynaptic_neuron_index % n)
while s[postsynaptic_neuron_index] < s_max:
potential_pre_index = np.random.randint(0, N_layer)
pre = (potential_pre_index // n, potential_pre_index % n)
# Commenting this 2 lines to allow for multapses
# if potential_pre_index not in existing_pre[
# postsynaptic_neuron_index]:
if formation_rule(pre, post, sigma, p):
s[postsynaptic_neuron_index] += 1
connections.append((potential_pre_index,
postsynaptic_neuron_index, g_max, delay))
# print " |"
return connections
def __call__(self, machine_graph, graph_mapper):
"""
:param machine_graph: the machine_graph whose edges are to be filtered
:param graph_mapper: the graph mapper between graphs
:return: a new graph mapper and machine graph
"""
new_machine_graph = MachineGraph(label=machine_graph.label)
new_graph_mapper = GraphMapper()
# create progress bar
progress = ProgressBar(
machine_graph.n_vertices +
machine_graph.n_outgoing_edge_partitions,
"Filtering edges")
# add the vertices directly, as they wont be pruned.
for vertex in progress.over(machine_graph.vertices, False):
self._add_vertex_to_new_graph(
vertex, graph_mapper, new_machine_graph, new_graph_mapper)
# start checking edges to decide which ones need pruning....
for partition in progress.over(machine_graph.outgoing_edge_partitions):
for edge in partition.edges:
if self._is_filterable(edge, graph_mapper):
logger.debug("this edge was pruned %s", edge)
continue
logger.debug("this edge was not pruned %s", edge)
self._add_edge_to_new_graph(
edge, partition, graph_mapper, new_machine_graph,
new_graph_mapper)
# returned the pruned graph and graph_mapper
return new_machine_graph, new_graph_mapper
def __call__(self,
transceiver,
machine,
app_id,
dsg_targets,
processor_to_app_data_base_address=None):
"""
:param machine: the python representation of the SpiNNaker machine
:param transceiver: the spinnman instance
:param app_id: the application ID of the simulation
:param dsg_targets: map of placement to file path
:return: map of placement and DSG data, and loaded data flag.
"""
# pylint: disable=too-many-arguments
if processor_to_app_data_base_address is None:
processor_to_app_data_base_address = dict()
# create a progress bar for end users
progress = ProgressBar(
dsg_targets, "Executing data specifications and loading data")
for (x, y, p), data_spec_file_path in \
progress.over(iteritems(dsg_targets)):
# write information for the memory map report
processor_to_app_data_base_address[x, y, p] = self._execute(
transceiver, machine, app_id, x, y, p, data_spec_file_path)
return processor_to_app_data_base_address
def __call__(self, report_default_directory, machine):
""" Creates a report that states where in SDRAM each region is.
:param report_default_directory: the folder where reports are written
:param machine: python representation of the machine
:rtype: None
"""
# create file path
directory_name = os.path.join(
report_default_directory, self.AREA_CODE_REPORT_NAME)
# create the progress bar for end users
progress_bar = ProgressBar(
len(machine.ethernet_connected_chips),
"Writing the board chip report")
# iterate over ethernet chips and then the chips on that board
with open(directory_name, "w") as writer:
for ethernet_connected_chip in \
progress_bar.over(machine.ethernet_connected_chips):
chips = machine.get_chips_on_board(ethernet_connected_chip)
writer.write(
"board with IP address : {} : has chips {}\n".format(
ethernet_connected_chip.ip_address, list(chips)))
def placement_report_without_application_graph_by_vertex(
report_folder, hostname, placements, machine_graph):
""" Generate report on the placement of vertices onto cores by vertex.
:param str report_folder: the folder to which the reports are being written
:param str hostname: the machine's hostname to which the placer worked on
:param Placements placements: the placements objects built by the placer.
:param MachineGraph machine_graph:
the machine graph generated by the end user
"""
# Cycle through all vertices, and for each cycle through its vertices.
# For each vertex, describe its core mapping.
file_name = os.path.join(report_folder, _PLACEMENT_VTX_SIMPLE_FILENAME)
time_date_string = time.strftime("%c")
try:
with open(file_name, "w") as f:
progress = ProgressBar(machine_graph.n_vertices,
"Generating placement report")
f.write(" Placement Information by Vertex\n")
f.write(" ===============================\n\n")
f.write("Generated: {} for target machine '{}'\n\n".format(
time_date_string, hostname))
for vertex in progress.over(machine_graph.vertices):
_write_one_vertex_machine_placement(f, vertex, placements)
except IOError:
logger.exception(
"Generate_placement_reports: Can't open file {} for"
" writing.", file_name)
def _run(self, placements, machine_graph):
""" Find routes between the edges with the allocated information,
placed in the given places
:param Placements placements: The placements of the edges
:param ~spinn_machine.Machine machine:
The machine through which the routes are to be found
:param MachineGraph machine_graph: the machine_graph object
:param bool use_progress_bar: whether to show a progress bar
:return: The discovered routes
:rtype: MulticastRoutingTableByPartition
:raise PacmanRoutingException:
If something goes wrong with the routing
"""
nodes_info = self._initiate_node_info()
tables = self._initiate_dijkstra_tables()
self._update_all_weights(nodes_info)
# each vertex represents a core in the board
progress = ProgressBar(placements.n_placements,
"Creating routing entries")
for placement in progress.over(placements.placements):
self._route(placement, placements, machine_graph, nodes_info,
tables)
return self._routing_paths
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(
all_core_subsets, CPUState.IDLE)
if error_cores or watchdog_cores or idle_cores:
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(
error_cores.values(), watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
self._update_provenance(txrx, total_processors, processors_completed,
all_core_subsets, app_id, progress)
progress.end()
def __call__(self, report_folder, connection_holder, dsg_targets):
""" Convert synaptic matrix for every application edge.
"""
# Update the print options to display everything
print_opts = numpy.get_printoptions()
numpy.set_printoptions(threshold=numpy.nan)
if dsg_targets is None:
raise SynapticConfigurationException(
"dsg_targets should not be none, used as a check for "
"connection holder data to be generated")
# generate folder for synaptic reports
top_level_folder = os.path.join(report_folder, _DIRNAME)
if not os.path.exists(top_level_folder):
os.mkdir(top_level_folder)
# create progress bar
progress = ProgressBar(connection_holder.keys(),
"Generating synaptic matrix reports")
# for each application edge, write matrix in new file
for edge, _ in progress.over(connection_holder.keys()):
# only write matrix's for edges which have matrix's
if isinstance(edge, ProjectionApplicationEdge):
# figure new file name
file_name = os.path.join(
top_level_folder, _TMPL_FILENAME.format(edge.label))
self._write_file(file_name, connection_holder, edge)
# Reset the print options
numpy.set_printoptions(**print_opts)
def __call__(self, router_tables, target_length=None):
# build storage
compressed_pacman_router_tables = MulticastRoutingTables()
# create progress bar
progress = ProgressBar(router_tables.routing_tables,
"Compressing routing Tables")
# compress each router
for router_table in progress.over(router_tables.routing_tables):
# convert to rig format
entries = self._convert_to_mundy_format(router_table)
# compress the router entries
compressed_router_table_entries = \
rigs_compressor.minimise(entries, target_length)
# convert back to pacman model
compressed_pacman_table = self._convert_to_pacman_router_table(
compressed_router_table_entries, router_table.x,
router_table.y)
# add to new compressed routing tables
compressed_pacman_router_tables.add_routing_table(
compressed_pacman_table)
# return
return compressed_pacman_router_tables
def __call__(self,
transceiver,
placements,
has_ran,
provenance_data_objects=None):
"""
:param transceiver: the SpiNNMan interface object
:param placements: The placements of the vertices
:param has_ran: token that states that the simulation has ran
"""
if not has_ran:
raise ConfigurationException(
"This function has been called before the simulation has ran."
" This is deemed an error, please rectify and try again")
if provenance_data_objects is not None:
prov_items = provenance_data_objects
else:
prov_items = list()
progress = ProgressBar(placements.n_placements,
"Getting provenance data")
# retrieve provenance data from any cores that provide data
for placement in progress.over(placements.placements):
if isinstance(placement.vertex,
AbstractProvidesProvenanceDataFromMachine):
# get data
prov_items.extend(
placement.vertex.get_provenance_data_from_machine(
transceiver, placement))
return prov_items
def __call__(
self, placements, hostname,
report_default_directory, write_text_specs,
machine, graph_mapper=None, placement_order=None):
"""
:param placements: placements of machine graph to cores
:param hostname: SpiNNaker machine name
:param report_default_directory: the location where reports are stored
:param write_text_specs:\
True if the textual version of the specification is to be written
:param machine: the python representation of the SpiNNaker machine
:param graph_mapper:\
the mapping between application and machine graph
:param placement:\
the optional order in which placements should be examined
:return: DSG targets (map of placement tuple and filename)
"""
# pylint: disable=too-many-arguments, too-many-locals
# pylint: disable=attribute-defined-outside-init
self._machine = machine
self._hostname = hostname
self._report_dir = report_default_directory
self._write_text = write_text_specs
# iterate though vertices and call generate_data_spec for each
# vertex
targets = DataSpecificationTargets(machine, self._report_dir)
if placement_order is None:
placement_order = placements.placements
progress = ProgressBar(
placements.n_placements, "Generating data specifications")
vertices_to_reset = list()
for placement in progress.over(placement_order):
# Try to generate the data spec for the placement
generated = self.__generate_data_spec_for_vertices(
placement, placement.vertex, targets)
if generated and isinstance(
placement.vertex, AbstractRewritesDataSpecification):
vertices_to_reset.append(placement.vertex)
# If the spec wasn't generated directly, and there is an
# application vertex, try with that
if not generated and graph_mapper is not None:
associated_vertex = graph_mapper.get_application_vertex(
placement.vertex)
generated = self.__generate_data_spec_for_vertices(
placement, associated_vertex, targets)
if generated and isinstance(
associated_vertex, AbstractRewritesDataSpecification):
vertices_to_reset.append(associated_vertex)
# Ensure that the vertices know their regions have been reloaded
for vertex in vertices_to_reset:
vertex.mark_regions_reloaded()
return targets
def __call__(self, fixed_routes, transceiver, app_id):
progress_bar = ProgressBar(
total_number_of_things_to_do=len(fixed_routes),
string_describing_what_being_progressed="loading fixed routes")
for chip_x, chip_y in progress_bar.over(fixed_routes.keys()):
transceiver.load_fixed_route(
chip_x, chip_y, fixed_routes[(chip_x, chip_y)], app_id)
def __call__(self, router_tables, app_id, transceiver, machine):
progress = ProgressBar(router_tables.routing_tables,
"Loading routing data onto the machine")
# load each router table that is needed for the application to run into
# the chips SDRAM
for table in progress.over(router_tables.routing_tables):
if (not machine.get_chip_at(table.x, table.y).virtual
and table.multicast_routing_entries):
transceiver.load_multicast_routes(
table.x, table.y, table.multicast_routing_entries,
app_id=app_id)
def __call__(self, machine_graph, machine, plan_n_timesteps):
"""
:param machine_graph: The machine_graph to place
:type machine_graph:\
:py:class:`pacman.model.graphs.machine.MachineGraph`
:param machine:\
The machine with respect to which to partition the application\
graph
:type machine: :py:class:`spinn_machine.Machine`
:param plan_n_timesteps: number of timesteps to plan for
:type plan_n_timesteps: int
:return: A set of placements
:rtype: :py:class:`pacman.model.placements.Placements`
:raise pacman.exceptions.PacmanPlaceException: \
If something goes wrong with the placement
"""
# check that the algorithm can handle the constraints
self._check_constraints(machine_graph.vertices)
# Sort the vertices into those with and those without
# placement constraints
placements = Placements()
constrained = list()
unconstrained = set()
for vertex in machine_graph.vertices:
if locate_constraints_of_type(
vertex.constraints, AbstractPlacerConstraint):
constrained.append(vertex)
else:
unconstrained.add(vertex)
# Iterate over constrained vertices and generate placements
progress = ProgressBar(
machine_graph.n_vertices, "Placing graph vertices")
resource_tracker = ResourceTracker(
machine, plan_n_timesteps, self._generate_radial_chips(machine))
constrained = sort_vertices_by_known_constraints(constrained)
for vertex in progress.over(constrained, False):
self._place_vertex(vertex, resource_tracker, machine, placements)
while unconstrained:
# Place the subgraph with the overall most connected vertex
max_connected_vertex = self._find_max_connected_vertex(
unconstrained, machine_graph)
self._place_unconstrained_subgraph(
max_connected_vertex, machine_graph, unconstrained,
machine, placements, resource_tracker, progress)
# finished, so stop progress bar and return placements
progress.end()
return placements
def __get_projection_data(
self, data_to_get, pre_vertex, post_vertex, connection_holder,
handle_time_out_configuration):
# pylint: disable=too-many-arguments, too-many-locals
ctl = self._spinnaker_control
# if using extra monitor functionality, locate extra data items
if ctl.get_generated_output("UsingAdvancedMonitorSupport"):
extra_monitors = ctl.get_generated_output(
"MemoryExtraMonitorVertices")
receivers = ctl.get_generated_output(
"MemoryMCGatherVertexToEthernetConnectedChipMapping")
extra_monitor_placements = ctl.get_generated_output(
"MemoryExtraMonitorToChipMapping")
else:
extra_monitors = None
receivers = None
extra_monitor_placements = None
edges = ctl.graph_mapper.get_machine_edges(self._projection_edge)
progress = ProgressBar(
edges, "Getting {}s for projection between {} and {}".format(
data_to_get, pre_vertex.label, post_vertex.label))
for edge in progress.over(edges):
placement = ctl.placements.get_placement_of_vertex(
edge.post_vertex)
# if using extra monitor data extractor find local receiver
if extra_monitors is not None:
receiver = helpful_functions.locate_extra_monitor_mc_receiver(
placement_x=placement.x, placement_y=placement.y,
machine=ctl.machine,
packet_gather_cores_to_ethernet_connection_map=receivers)
sender_extra_monitor_core = extra_monitor_placements[
placement.x, placement.y]
sender_monitor_place = ctl.placements.get_placement_of_vertex(
sender_extra_monitor_core)
else:
receiver = None
sender_monitor_place = None
connections = post_vertex.get_connections_from_machine(
ctl.transceiver, placement, edge, ctl.graph_mapper,
ctl.routing_infos, self._synapse_information,
ctl.machine_time_step, extra_monitors is not None,
ctl.placements, receiver, sender_monitor_place,
extra_monitors, handle_time_out_configuration,
ctl.fixed_routes)
if connections is not None:
connection_holder.add_connections(connections)
connection_holder.finish()
def __call__(self, machine_graph, placements, buffer_manager):
# Count the regions to be read
n_regions_to_read, recording_placements = self._count_regions(
machine_graph, placements)
# Read back the regions
progress = ProgressBar(
n_regions_to_read, "Extracting buffers from the last run")
try:
buffer_manager.get_data_for_placements(
recording_placements, progress)
finally:
progress.end()
def synapse_expander(
app_graph, graph_mapper, placements, transceiver,
provenance_file_path, executable_finder):
""" Run the synapse expander - needs to be done after data has been loaded
"""
synapse_expander = executable_finder.get_executable_path(SYNAPSE_EXPANDER)
delay_expander = executable_finder.get_executable_path(DELAY_EXPANDER)
progress = ProgressBar(len(app_graph.vertices) + 2, "Expanding Synapses")
# Find the places where the synapse expander and delay receivers should run
expander_cores = ExecutableTargets()
for vertex in progress.over(app_graph.vertices, finish_at_end=False):
# Find population vertices
if isinstance(
vertex, (AbstractPopulationVertex, DelayExtensionVertex)):
# Add all machine vertices of the population vertex to ones
# that need synapse expansion
for m_vertex in graph_mapper.get_machine_vertices(vertex):
vertex_slice = graph_mapper.get_slice(m_vertex)
if vertex.gen_on_machine(vertex_slice):
placement = placements.get_placement_of_vertex(m_vertex)
if isinstance(vertex, AbstractPopulationVertex):
binary = synapse_expander
else:
binary = delay_expander
expander_cores.add_processor(
binary, placement.x, placement.y, placement.p)
# Launch the delay receivers
expander_app_id = transceiver.app_id_tracker.get_new_id()
transceiver.execute_application(expander_cores, expander_app_id)
progress.update()
# Wait for everything to finish
finished = False
try:
transceiver.wait_for_cores_to_be_in_state(
expander_cores.all_core_subsets, expander_app_id,
[CPUState.FINISHED])
progress.update()
finished = True
_extract_iobuf(expander_cores, transceiver, provenance_file_path)
progress.end()
except Exception:
logger.exception("Synapse expander has failed")
_handle_failure(
expander_cores, transceiver, provenance_file_path)
finally:
transceiver.stop_application(expander_app_id)
transceiver.app_id_tracker.free_id(expander_app_id)
if not finished:
raise SpynnakerException(
"The synapse expander failed to complete")
def __call__(
self, machine, n_machine_time_steps, n_samples_per_recording,
sampling_frequency, time_scale_factor, machine_time_step,
pre_allocated_resources=None):
"""
:param pre_allocated_resources: other preallocated resources
:param machine: the SpiNNaker machine as discovered
:param n_machine_time_steps: the number of machine\
time steps used by the simulation during this phase
:param n_samples_per_recording: how many samples between record entries
:param sampling_frequency: the frequency of sampling
:param time_scale_factor: the time scale factor
:param machine_time_step: the machine time step
:return: preallocated resources
"""
# pylint: disable=too-many-arguments
progress_bar = ProgressBar(
machine.n_chips, "Preallocating resources for chip power monitor")
# store how much SDRAM the power monitor uses per core
resources = ChipPowerMonitorMachineVertex.get_resources(
n_machine_time_steps=n_machine_time_steps,
n_samples_per_recording=n_samples_per_recording,
sampling_frequency=sampling_frequency,
time_scale_factor=time_scale_factor,
time_step=machine_time_step)
# for every Ethernet connected chip, get the resources needed by the
# live packet gatherers
sdrams = list()
cores = list()
for chip in progress_bar.over(machine.chips):
sdrams.append(
SpecificChipSDRAMResource(chip, resources.sdram.get_value()))
cores.append(CoreResource(chip, 1))
# create preallocated resource container
cpm_pre_allocated_resource_container = PreAllocatedResourceContainer(
specific_sdram_usage=sdrams, core_resources=cores)
# add other preallocated resources
if pre_allocated_resources is not None:
cpm_pre_allocated_resource_container.extend(
pre_allocated_resources)
# return preallocated resources
return cpm_pre_allocated_resource_container
def __call__(self, app_id, txrx, executable_types):
total_processors = \
len(executable_types[ExecutableType.USES_SIMULATION_INTERFACE])
all_core_subsets = \
executable_types[ExecutableType.USES_SIMULATION_INTERFACE]
progress = ProgressBar(
total_processors,
"Turning off all the cores within the simulation")
# check that the right number of processors are finished
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
finished_cores = processors_finished
while processors_finished != total_processors:
if processors_finished > finished_cores:
progress.update(processors_finished - finished_cores)
finished_cores = processors_finished
processors_rte = txrx.get_core_state_count(
app_id, CPUState.RUN_TIME_EXCEPTION)
processors_watchdogged = txrx.get_core_state_count(
app_id, CPUState.WATCHDOG)
if processors_rte > 0 or processors_watchdogged > 0:
raise ExecutableFailedToStopException(
"{} of {} processors went into an error state when"
" shutting down".format(
processors_rte + processors_watchdogged,
total_processors))
successful_cores_finished = txrx.get_cores_in_state(
all_core_subsets, CPUState.FINISHED)
for core_subset in all_core_subsets:
for processor in core_subset.processor_ids:
if not successful_cores_finished.is_core(
core_subset.x, core_subset.y, processor):
self._update_provenance_and_exit(
txrx, processor, core_subset)
time.sleep(0.5)
processors_finished = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
progress.end()
def __call__(
self, placements, graph, executable_finder, graph_mapper=None):
executable_targets = ExecutableTargets()
progress = ProgressBar(graph.n_vertices, "Finding binaries")
for vertex in progress.over(graph.vertices):
placement = placements.get_placement_of_vertex(vertex)
self._get_binary(
placement, vertex, executable_targets, executable_finder)
if graph_mapper is not None:
associated_vertex = graph_mapper.get_application_vertex(vertex)
self._get_binary(
placement, associated_vertex, executable_targets,
executable_finder)
return executable_targets
def __call__(self, machine_graph=None, application_graph=None,
graph_mapper=None):
# Generate an n_keys map for the graph and add constraints
n_keys_map = DictBasedMachinePartitionNKeysMap()
if machine_graph is None:
raise ConfigurationException(
"A machine graph is required for this mapper. "
"Please choose and try again")
if (application_graph is None) != (graph_mapper is None):
raise ConfigurationException(
"Can only do one graph. semantically doing 2 graphs makes no "
"sense. Please choose and try again")
if application_graph is not None:
# generate progress bar
progress = ProgressBar(
machine_graph.n_vertices,
"Getting number of keys required by each edge using "
"application graph")
# iterate over each partition in the graph
for vertex in progress.over(machine_graph.vertices):
partitions = machine_graph.\
get_outgoing_edge_partitions_starting_at_vertex(
vertex)
for partition in partitions:
if partition.traffic_type == EdgeTrafficType.MULTICAST:
self._process_application_partition(
partition, n_keys_map, graph_mapper)
else:
# generate progress bar
progress = ProgressBar(
machine_graph.n_vertices,
"Getting number of keys required by each edge using "
"machine graph")
for vertex in progress.over(machine_graph.vertices):
partitions = machine_graph.\
get_outgoing_edge_partitions_starting_at_vertex(
vertex)
for partition in partitions:
if partition.traffic_type == EdgeTrafficType.MULTICAST:
self._process_machine_partition(partition, n_keys_map)
return n_keys_map
def __call__(
self, routing_infos, routing_table_by_partitions, machine):
"""
:param routing_infos:
:param routing_table_by_partitions:
:param machine:
"""
progress = ProgressBar(machine.n_chips, "Generating routing tables")
routing_tables = MulticastRoutingTables()
for chip in progress.over(machine.chips):
partitions_in_table = routing_table_by_partitions.\
get_entries_for_router(chip.x, chip.y)
if partitions_in_table:
routing_tables.add_routing_table(self._create_routing_table(
chip, partitions_in_table, routing_infos))
return routing_tables
def __call__(
self, placements, tags, txrx,
uses_advanced_monitors, report_folder, extra_monitor_cores=None,
extra_monitor_to_chip_mapping=None,
packet_gather_cores_to_ethernet_connection_map=None, machine=None,
fixed_routes=None, java_caller=None):
"""
:param placements:
:param tags:
:param txrx:
:param uses_advanced_monitors:
:param extra_monitor_cores:
:param extra_monitor_to_chip_mapping:
:param packet_gather_cores_to_ethernet_connection_map:
:param machine:
:param fixed_routes:
:param report_folder: The path where
the SQLite database holding the data will be placed,
and where any java provenance can be written.
:type report_folder: str
:return:
"""
# pylint: disable=too-many-arguments
progress = ProgressBar(placements.n_placements, "Initialising buffers")
# Create the buffer manager
buffer_manager = BufferManager(
placements=placements, tags=tags, transceiver=txrx,
extra_monitor_cores=extra_monitor_cores,
packet_gather_cores_to_ethernet_connection_map=(
packet_gather_cores_to_ethernet_connection_map),
extra_monitor_to_chip_mapping=extra_monitor_to_chip_mapping,
machine=machine, uses_advanced_monitors=uses_advanced_monitors,
fixed_routes=fixed_routes, report_folder=report_folder,
java_caller=java_caller)
for placement in progress.over(placements.placements):
if isinstance(placement.vertex, AbstractSendsBuffersFromHost):
if placement.vertex.buffering_input():
buffer_manager.add_sender_vertex(placement.vertex)
if isinstance(placement.vertex, AbstractReceiveBuffersToHost):
buffer_manager.add_receiving_vertex(placement.vertex)
return buffer_manager
def generate_machine_edges(machine_graph, graph_mapper, application_graph):
""" Generate the machine edges for the vertices in the graph
:param machine_graph: the machine graph to add edges to
:type machine_graph:\
:py:class:`pacman.model.graphs.machine.MachineGraph`
:param graph_mapper: the mapper graphs
:type graph_mapper:\
:py:class:`pacman.model.graphs.common.GraphMapper`
:param application_graph: the application graph to work with
:type application_graph:\
:py:class:`pacman.model.graphs.application.ApplicationGraph`
"""
# start progress bar
progress = ProgressBar(
machine_graph.n_vertices, "Partitioning graph edges")
# Partition edges according to vertex partitioning
for source_vertex in progress.over(machine_graph.vertices):
# For each out edge of the parent vertex...
vertex = graph_mapper.get_application_vertex(source_vertex)
application_outgoing_partitions = application_graph.\
get_outgoing_edge_partitions_starting_at_vertex(vertex)
for application_partition in application_outgoing_partitions:
for application_edge in application_partition.edges:
# create new partitions
for dest_vertex in graph_mapper.get_machine_vertices(
application_edge.post_vertex):
machine_edge = application_edge.create_machine_edge(
source_vertex, dest_vertex,
"machine_edge_for{}".format(application_edge.label))
machine_graph.add_edge(
machine_edge, application_partition.identifier)
# add constraints from the application partition
machine_partition = machine_graph.\
get_outgoing_edge_partition_starting_at_vertex(
source_vertex, application_partition.identifier)
machine_partition.add_constraints(
application_partition.constraints)
# update mapping object
graph_mapper.add_edge_mapping(
machine_edge, application_edge)
def __call__(self, machine_graph, machine, file_path):
"""
:param machine_graph: the machine graph
:param machine: the machine
"""
progress = ProgressBar(
machine_graph.n_vertices + 2, "creating JSON constraints")
json_obj = list()
self._add_monitor_core_reserve(json_obj)
progress.update()
self._add_extra_monitor_cores(json_obj, machine)
progress.update()
vertex_by_id = self._search_graph_for_placement_constraints(
json_obj, machine_graph, machine, progress)
with open(file_path, "w") as f:
json.dump(json_obj, f)
# validate the schema
file_format_schemas.validate(json_obj, "constraints.json")
# complete progress bar
progress.end()
return file_path, vertex_by_id
def allocate_chip_ids(self, machine, graph):
""" Go through the chips (real and virtual) and allocate keys for each
"""
progress = ProgressBar(
graph.n_vertices + machine.n_chips,
"Allocating virtual identifiers")
# allocate standard IDs for real chips
for x, y in progress.over(machine.chip_coordinates, False):
expected_chip_id = (x << 8) + y
self._allocate_elements(expected_chip_id, 1)
# allocate IDs for virtual chips
for vertex in progress.over(graph.vertices):
if isinstance(vertex, AbstractVirtualVertex):
x, y = self._assign_virtual_chip_info(
machine, self._get_link_data(machine, vertex))
vertex.set_virtual_chip_coordinates(x, y)
