def __call__(self, live_packet_gatherer_parameters, machine, machine_graph, application_graph=None, graph_mapper=None): """ Add LPG vertices on Ethernet connected chips as required. :param live_packet_gatherer_parameters:\ the Live Packet Gatherer parameters requested by the script :param machine: the SpiNNaker machine as discovered :param application_graph: the application graph :param machine_graph: the machine graph :return: mapping between LPG parameters and LPG vertex """ # pylint: disable=too-many-arguments # create progress bar progress = ProgressBar(machine.ethernet_connected_chips, string_describing_what_being_progressed=( "Adding Live Packet Gatherers to Graph")) # Keep track of the vertices added by parameters and board address lpg_params_to_vertices = defaultdict(dict) # for every Ethernet connected chip, add the gatherers required for chip in progress.over(machine.ethernet_connected_chips): for params in live_packet_gatherer_parameters: if (params.board_address is None or params.board_address == chip.ip_address): lpg_params_to_vertices[params][chip.x, chip.y] = \ self._add_lpg_vertex(application_graph, graph_mapper, machine_graph, chip, params) return lpg_params_to_vertices
def _run_for_core_subsets(self, core_subsets, transceiver, provenance_file_path): progress = ProgressBar(len(core_subsets), "Extracting IOBUF from the machine") error_entries = list() warn_entries = list() # extract iobuf io_buffers = list(transceiver.get_iobuf(core_subsets)) # write iobuf for iobuf in io_buffers: file_name = os.path.join( provenance_file_path, "{}_{}_{}.txt".format(iobuf.x, iobuf.y, iobuf.p)) # set mode of the file based off if the file already exists mode = "w" if os.path.exists(file_name): mode = "a" # write iobuf to file. with open(file_name, mode) as writer: if mode == "a": writer.write("\n\n") writer.write(iobuf.iobuf) # check iobuf for errors for io_buffer in progress.over(io_buffers): self._check_iobuf_for_error(io_buffer, error_entries, warn_entries) return error_entries, warn_entries
def __call__(self, txrx, app_id, all_core_subsets): # check that the right number of processors are in sync processors_completed = txrx.get_core_state_count( app_id, CPUState.FINISHED) total_processors = len(all_core_subsets) left_to_do_cores = total_processors - processors_completed progress = ProgressBar( left_to_do_cores, "Forcing error cores to generate provenance data") error_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.RUN_TIME_EXCEPTION) watchdog_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.WATCHDOG) idle_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.IDLE) if error_cores or watchdog_cores or idle_cores: raise ConfigurationException( "Some cores have crashed. RTE cores {}, watch-dogged cores {}," " idle cores {}".format(error_cores.values(), watchdog_cores.values(), idle_cores.values())) # check that all cores are in the state FINISHED which shows that # the core has received the message and done provenance updating self._update_provenance(txrx, total_processors, processors_completed, all_core_subsets, app_id, progress) progress.end()
def __call__(self, router_tables, app_id, transceiver, machine): """ :param ~.MulticastRoutingTables router_tables: :param int app_id: :param ~.Transceiver transceiver: :param ~.Machine machine: """ routing_tables = list(router_tables.routing_tables) progress = ProgressBar(routing_tables, "Preparing Routing Tables") # Clear the routing table for each router that needs to be set up # and set up the diagnostics for router_table in progress.over(routing_tables): if not machine.get_chip_at(router_table.x, router_table.y).virtual: transceiver.clear_multicast_routes( router_table.x, router_table.y) transceiver.clear_router_diagnostic_counters( router_table.x, router_table.y) # set the router diagnostic for user 3 to catch local default # routed packets. This can only occur when the source router # has no router entry, and therefore should be detected a bad # dropped packet. self._set_router_diagnostic_filters( router_table.x, router_table.y, transceiver)
def __call__(self, machine, extra_monitor_cores, placements): """ :param ~spinn_machine.Machine machine: :param dict(tuple(int,int),ExtraMonitorSupportMachineVertex) \ extra_monitor_cores: :param ~pacman.model.placements.Placements placements: :rtype: tuple(MulticastRoutingTables, dict(tuple(int,int),int), dict(tuple(int,int),int)) """ # pylint: disable=attribute-defined-outside-init self._machine = machine self._placements = placements self._monitors = extra_monitor_cores self._routing_tables = MulticastRoutingTables() self._key_to_destination_map = dict() self._time_out_keys_by_board = dict() # create progress bar progress = ProgressBar( machine.ethernet_connected_chips, "Generating routing tables for data in system processes") for ethernet_chip in progress.over(machine.ethernet_connected_chips): tree = self._generate_routing_tree(ethernet_chip) self._add_routing_entries(ethernet_chip, tree) return (self._routing_tables, self._key_to_destination_map, self._time_out_keys_by_board)
def get_spikes(self, label, buffer_manager, region, placements, graph_mapper, application_vertex, base_key_function, machine_time_step): # pylint: disable=too-many-arguments results = list() missing = [] ms_per_tick = machine_time_step / 1000.0 vertices = graph_mapper.get_machine_vertices(application_vertex) progress = ProgressBar(vertices, "Getting spikes for {}".format(label)) for vertex in progress.over(vertices): placement = placements.get_placement_of_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) # Read the spikes raw_spike_data, data_missing = \ buffer_manager.get_data_by_placement(placement, region) if data_missing: missing.append(placement) self._process_spike_data( vertex_slice, raw_spike_data, ms_per_tick, base_key_function(vertex), results) if missing: missing_str = recording_utils.make_missing_string(missing) logger.warning( "Population {} is missing spike data in region {} from the" " following cores: {}", label, region, missing_str) if not results: return numpy.empty(shape=(0, 2)) result = numpy.vstack(results) return result[numpy.lexsort((result[:, 1], result[:, 0]))]
def placement_report_without_application_graph_by_core(report_folder, hostname, placements, machine): """ Generate report on the placement of vertices onto cores by core. :param str report_folder: the folder to which the reports are being written :param str hostname: the machine's hostname to which the placer worked on :param ~spinn_machine.Machine machine: the SpiNNaker machine object :param Placements placements: the placements objects built by the placer. """ # File 2: Placement by core. # Cycle through all chips and by all cores within each chip. # For each core, display what is held on it. file_name = os.path.join(report_folder, _PLACEMENT_CORE_SIMPLE_FILENAME) time_date_string = time.strftime("%c") try: with open(file_name, "w") as f: progress = ProgressBar(machine.chips, "Generating placement by core report") f.write(" Placement Information by Core\n") f.write(" =============================\n\n") f.write("Generated: {}".format(time_date_string)) f.write(" for target machine '{}'".format(hostname)) f.write("\n\n") for chip in progress.over(machine.chips): _write_one_chip_machine_placement(f, chip, placements) except IOError: logger.exception( "Generate_placement_reports: Can't open file {} for " "writing.", file_name)
def __call__( self, transceiver, tags=None, iptags=None, reverse_iptags=None): """ :param tags: the tags object which contains IP and reverse IP tags. could be none if these are being given in separate lists :param iptags: a list of IP tags, given when tags is none :param reverse_iptags: a list of reverse IP tags when tags is none. :param transceiver: the transceiver object """ # clear all the tags from the Ethernet connection, as nothing should # be allowed to use it (no two apps should use the same Ethernet # connection at the same time) progress = ProgressBar(MAX_TAG_ID, "Clearing tags") for tag_id in progress.over(range(MAX_TAG_ID)): transceiver.clear_ip_tag(tag_id) # Use tags object to supply tag info if it is supplied if tags is not None: iptags = list(tags.ip_tags) reverse_iptags = list(tags.reverse_ip_tags) # Load the IP tags and the Reverse IP tags progress = ProgressBar( len(iptags) + len(reverse_iptags), "Loading Tags") self.load_iptags(iptags, transceiver, progress) self.load_reverse_iptags(reverse_iptags, transceiver, progress) progress.end()
def __call__(self, report_folder, connection_holder, dsg_targets): """ Convert synaptic matrix for every application edge. """ # Update the print options to display everything print_opts = numpy.get_printoptions() numpy.set_printoptions(threshold=numpy.nan) if dsg_targets is None: raise SynapticConfigurationException( "dsg_targets should not be none, used as a check for " "connection holder data to be generated") # generate folder for synaptic reports top_level_folder = os.path.join(report_folder, _DIRNAME) if not os.path.exists(top_level_folder): os.mkdir(top_level_folder) # create progress bar progress = ProgressBar(connection_holder.keys(), "Generating synaptic matrix reports") # for each application edge, write matrix in new file for edge, _ in progress.over(connection_holder.keys()): # only write matrix's for edges which have matrix's if isinstance(edge, ProjectionApplicationEdge): # figure new file name file_name = os.path.join(top_level_folder, _TMPL_FILENAME.format(edge.label)) self._write_file(file_name, connection_holder, edge) # Reset the print options numpy.set_printoptions(**print_opts)
def __call__( self, live_packet_gatherer_parameters, machine, machine_graph, application_graph=None, graph_mapper=None): """ Add LPG vertices on Ethernet connected chips as required. :param live_packet_gatherer_parameters:\ the Live Packet Gatherer parameters requested by the script :param machine: the SpiNNaker machine as discovered :param application_graph: the application graph :param machine_graph: the machine graph :return: mapping between LPG parameters and LPG vertex """ # pylint: disable=too-many-arguments # create progress bar progress = ProgressBar( machine.ethernet_connected_chips, string_describing_what_being_progressed=( "Adding Live Packet Gatherers to Graph")) # Keep track of the vertices added by parameters and board address lpg_params_to_vertices = defaultdict(dict) # for every Ethernet connected chip, add the gatherers required for chip in progress.over(machine.ethernet_connected_chips): for params in live_packet_gatherer_parameters: if (params.board_address is None or params.board_address == chip.ip_address): lpg_params_to_vertices[params][chip.x, chip.y] = \ self._add_lpg_vertex(application_graph, graph_mapper, machine_graph, chip, params) return lpg_params_to_vertices
def __call__( self, transceiver, placements, provenance_file_path, run_time_ms, machine_time_step): """ :param transceiver: the SpiNNMan interface object :param placements: The placements of the vertices :param has_ran: token that states that the simulation has ran :param provenance_file_path: The location to store the profile data :param run_time_ms: runtime in ms :param machine_time_step: machine time step in ms """ # pylint: disable=too-many-arguments machine_time_step_ms = machine_time_step // 1000 progress = ProgressBar( placements.n_placements, "Getting profile data") # retrieve provenance data from any cores that provide data for placement in progress.over(placements.placements): if isinstance(placement.vertex, AbstractHasProfileData): # get data profile_data = placement.vertex.get_profile_data( transceiver, placement) if profile_data.tags: self._write(placement, profile_data, run_time_ms, machine_time_step_ms, provenance_file_path)
def __call__( self, transceiver, placements, provenance_file_path, machine_time_step): """ :param ~.Transceiver transceiver: :param ~.Placements placements: :param str provenance_file_path: :param int machine_time_step: """ # pylint: disable=too-many-arguments machine_time_step_ms = ( float(machine_time_step) / MICRO_TO_MILLISECOND_CONVERSION) progress = ProgressBar( placements.n_placements, "Getting profile data") # retrieve provenance data from any cores that provide data for placement in progress.over(placements.placements): if isinstance(placement.vertex, AbstractHasProfileData): # get data profile_data = placement.vertex.get_profile_data( transceiver, placement) if profile_data.tags: self._write(placement, profile_data, machine_time_step_ms, provenance_file_path)
def __call__(self, machine_graph, graph_mapper): """ :param machine_graph: the machine_graph whose edges are to be filtered :param graph_mapper: the graph mapper between graphs :return: a new graph mapper and machine graph """ new_machine_graph = MachineGraph(label=machine_graph.label) new_graph_mapper = GraphMapper() # create progress bar progress = ProgressBar( machine_graph.n_vertices + machine_graph.n_outgoing_edge_partitions, "Filtering edges") # add the vertices directly, as they wont be pruned. for vertex in progress.over(machine_graph.vertices, False): self._add_vertex_to_new_graph(vertex, graph_mapper, new_machine_graph, new_graph_mapper) # start checking edges to decide which ones need pruning.... for partition in progress.over(machine_graph.outgoing_edge_partitions): for edge in partition.edges: if self._is_filterable(edge, graph_mapper): logger.debug("this edge was pruned %s", edge) continue logger.debug("this edge was not pruned %s", edge) self._add_edge_to_new_graph(edge, partition, graph_mapper, new_machine_graph, new_graph_mapper) # returned the pruned graph and graph_mapper return new_machine_graph, new_graph_mapper
def __call__(self, machine_graph, machine, plan_n_timesteps): """ Place each vertex in a machine graph on a core in the machine. :param machine_graph: The machine_graph to place :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param machine: A SpiNNaker machine object. :type machine: :py:class:`spinn_machine.Machine` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :return placements: Placements of vertices on the machine :rtype :py:class:`pacman.model.placements.Placements` """ # check that the algorithm can handle the constraints ResourceTracker.check_constraints(machine_graph.vertices) placements = Placements() vertices = sort_vertices_by_known_constraints(machine_graph.vertices) # Iterate over vertices and generate placements progress = ProgressBar(machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_random_chips(machine)) vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph) vertices_placed = set() for vertex in progress.over(vertices): if vertex not in vertices_placed: vertices_placed.update(self._place_vertex( vertex, resource_tracker, machine, placements, vertices_on_same_chip)) return placements
def __call__(self, router_tables): tables = MulticastRoutingTables() previous_masks = dict() progress = ProgressBar( len(router_tables.routing_tables) * 2, "Compressing Routing Tables") # Create all masks without holes allowed_masks = [_32_BITS - ((2**i) - 1) for i in range(33)] # Check that none of the masks have "holes" e.g. 0xFFFF0FFF has a hole for router_table in router_tables.routing_tables: for entry in router_table.multicast_routing_entries: if entry.mask not in allowed_masks: raise PacmanRoutingException( "Only masks without holes are allowed in tables for" " MallocBasedRouteMerger (disallowed mask={})".format( hex(entry.mask))) for router_table in progress.over(router_tables.routing_tables): new_table = self._merge_routes(router_table, previous_masks) tables.add_routing_table(new_table) n_entries = len([ entry for entry in new_table.multicast_routing_entries if not entry.defaultable ]) print("Reduced from {} to {}".format( len(router_table.multicast_routing_entries), n_entries)) if n_entries > 1023: raise PacmanRoutingException( "Cannot make table small enough: {} entries".format( n_entries)) return tables
def __call__(self, router_tables): tables = MulticastRoutingTables() previous_masks = dict() progress = ProgressBar( len(router_tables.routing_tables) * 2, "Compressing Routing Tables") # Create all masks without holes allowed_masks = [_32_BITS - ((2 ** i) - 1) for i in range(33)] # Check that none of the masks have "holes" e.g. 0xFFFF0FFF has a hole for router_table in router_tables.routing_tables: for entry in router_table.multicast_routing_entries: if entry.mask not in allowed_masks: raise PacmanRoutingException( "Only masks without holes are allowed in tables for" " BasicRouteMerger (disallowed mask={})".format( hex(entry.mask))) for router_table in progress.over(router_tables.routing_tables): new_table = self._merge_routes(router_table, previous_masks) tables.add_routing_table(new_table) n_entries = len([ entry for entry in new_table.multicast_routing_entries if not entry.defaultable]) # print("Reduced from {} to {}".format( # len(router_table.multicast_routing_entries), n_entries)) if n_entries > 1023: raise PacmanRoutingException( "Cannot make table small enough: {} entries".format( n_entries)) return tables
def __call__(self, machine_graph, machine, plan_n_timesteps): """ Place each vertex in a machine graph on a core in the machine. :param MachineGraph machine_graph: The machine_graph to place :param ~spinn_machine.Machine machine: A SpiNNaker machine object. :param int plan_n_timesteps: number of timesteps to plan for :return placements: Placements of vertices on the machine :rtype: Placements """ # check that the algorithm can handle the constraints ResourceTracker.check_constraints(machine_graph.vertices) placements = Placements() vertices = sort_vertices_by_known_constraints(machine_graph.vertices) # Iterate over vertices and generate placements progress = ProgressBar(machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_random_chips(machine)) vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph) vertices_placed = set() for vertex in progress.over(vertices): if vertex not in vertices_placed: vertices_placed.update( self._place_vertex(vertex, resource_tracker, machine, placements, vertices_on_same_chip)) return placements
def __call__(self, router_tables, target_length=None): # build storage compressed_pacman_router_tables = MulticastRoutingTables() # create progress bar progress = ProgressBar( router_tables.routing_tables, "Compressing routing Tables") # compress each router for router_table in progress.over(router_tables.routing_tables): # convert to rig format entries = self._convert_to_mundy_format(router_table) # compress the router entries compressed_router_table_entries = \ rigs_compressor.minimise(entries, target_length) # convert back to pacman model compressed_pacman_table = self._convert_to_pacman_router_table( compressed_router_table_entries, router_table.x, router_table.y) # add to new compressed routing tables compressed_pacman_router_tables.add_routing_table( compressed_pacman_table) # return return compressed_pacman_router_tables
def partitioner_report(report_folder, hostname, graph): """ Generate report on the placement of vertices onto cores. :param str report_folder: the folder to which the reports are being written :param str hostname: the machine's hostname to which the placer worked on :param ApplicationGraph graph: the app graph """ # Cycle through all vertices, and for each cycle through its vertices. # For each vertex, describe its core mapping. file_name = os.path.join(report_folder, _PARTITIONING_FILENAME) time_date_string = time.strftime("%c") try: with open(file_name, "w") as f: progress = ProgressBar(graph.n_vertices, "Generating partitioner report") f.write(" Partitioning Information by Vertex\n") f.write(" ===============================\n\n") f.write("Generated: {} for target machine '{}'\n\n".format( time_date_string, hostname)) for vertex in progress.over(graph.vertices): _write_one_vertex_partition(f, vertex) except IOError: logger.exception( "Generate_placement_reports: Can't open file {} for" " writing.", file_name)
def __call__(self, report_default_directory, dsg_targets, transceiver): """ Creates a report that states where in SDRAM each region is \ (read from machine) :param report_default_directory: the folder where reports are written :param dsg_targets: the map between placement and file writer :param transceiver: the spinnMan instance :rtype: None """ directory_name = os.path.join( report_default_directory, MEM_MAP_SUBDIR_NAME) if not os.path.exists(directory_name): os.makedirs(directory_name) progress = ProgressBar(dsg_targets, "Writing memory map reports") for (x, y, p) in progress.over(dsg_targets): file_name = os.path.join( directory_name, MEM_MAP_FILENAME.format(x, y, p)) try: with open(file_name, "w") as f: self._describe_mem_map(f, transceiver, x, y, p) except IOError: logger.exception("Generate_placement_reports: Can't open file" " {} for writing.", file_name)
def get_spikes(self, label, buffer_manager, region, placements, graph_mapper, application_vertex, base_key_function, machine_time_step): # pylint: disable=too-many-arguments results = list() missing = [] ms_per_tick = machine_time_step / 1000.0 vertices = graph_mapper.get_machine_vertices(application_vertex) progress = ProgressBar(vertices, "Getting spikes for {}".format(label)) for vertex in progress.over(vertices): placement = placements.get_placement_of_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) # Read the spikes raw_spike_data, data_missing = \ buffer_manager.get_data_by_placement(placement, region) if data_missing: missing.append(placement) self._process_spike_data(vertex_slice, raw_spike_data, ms_per_tick, base_key_function(vertex), results) if missing: missing_str = recording_utils.make_missing_string(missing) logger.warning( "Population {} is missing spike data in region {} from the" " following cores: {}", label, region, missing_str) if not results: return numpy.empty(shape=(0, 2)) result = numpy.vstack(results) return result[numpy.lexsort((result[:, 1], result[:, 0]))]
def _fill_in_connection_data(transceiver, expanded_pop_vertices, placements): """ Once expander has run, fill in the connection data :rtype: None """ ctl = globals_variables.get_simulator() use_extra_monitors = False progress_bar = ProgressBar( len(expanded_pop_vertices), "reading back synaptic data required for connection holders") for vertex in progress_bar.over(expanded_pop_vertices): conn_holders = vertex.get_connection_holders() for (app_edge, synapse_info), conn_holder_list in iteritems(conn_holders): # Only do this if this synapse_info has been generated # on the machine using the expander if synapse_info.may_generate_on_machine(): machine_edges = app_edge.machine_edges for machine_edge in machine_edges: placement = placements.get_placement_of_vertex( machine_edge.post_vertex) conns = vertex.get_connections_from_machine( transceiver, placement, machine_edge, ctl.routing_infos, synapse_info, ctl.machine_time_step, use_extra_monitors) for conn_holder in conn_holder_list: conn_holder.add_connections(conns) for conn_holder in conn_holder_list: conn_holder.finish()
def _ner_route(machine_graph, machine, placements, vector_to_nodes): """ Performs routing using rig algorithm :param MachineGraph machine_graph: :param ~spinn_machine.Machine machine: :param Placements placements: :return: :rtype: MulticastRoutingTableByPartition """ routing_tables = MulticastRoutingTableByPartition() progress_bar = ProgressBar(len(machine_graph.vertices), "Routing") for source_vertex in progress_bar.over(machine_graph.vertices): # handle the vertex edges for partition in machine_graph.\ get_multicast_edge_partitions_starting_at_vertex( source_vertex): post_vertexes = list(e.post_vertex for e in partition.edges) routing_tree = _do_route(source_vertex, post_vertexes, machine, placements, vector_to_nodes) incoming_processor = placements.get_placement_of_vertex( partition.pre_vertex).p _convert_a_route(routing_tables, partition, incoming_processor, None, routing_tree) progress_bar.end() return routing_tables
def __call__(self, machine_graph, machine, plan_n_timesteps): """ Place a machine_graph so that each vertex is placed on a core :param machine_graph: The machine_graph to place :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param machine:\ The machine with respect to which to partition the application\ graph :type machine: :py:class:`spinn_machine.Machine` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :return: A set of placements :rtype: :py:class:`pacman.model.placements.Placements` :raise pacman.exceptions.PacmanPlaceException: \ If something goes wrong with the placement """ # check that the algorithm can handle the constraints ResourceTracker.check_constraints(machine_graph.vertices) placements = Placements() vertices = sort_vertices_by_known_constraints(machine_graph.vertices) # Iterate over vertices and generate placements progress = ProgressBar(vertices, "Placing graph vertices") resource_tracker = ResourceTracker(machine, plan_n_timesteps) for vertex in progress.over(vertices): # Create and store a new placement anywhere on the board (x, y, p, _, _) = resource_tracker.allocate_constrained_resources( vertex.resources_required, vertex.constraints, None) placement = Placement(vertex, x, y, p) placements.add_placement(placement) return placements
def __call__(self, machine_graph, placements, n_keys_map): """ :param MachineGraph machine_graph: The graph to allocate the routing info for :param Placements placements: The placements of the vertices :param AbstractMachinePartitionNKeysMap n_keys_map: A map between the edges and the number of keys required by the edges :return: The routing information :rtype: tuple(RoutingInfo, AbstractMulticastRoutingTable) :raise PacmanRouteInfoAllocationException: If something goes wrong with the allocation """ # check that this algorithm supports the constraints put onto the # partitions check_algorithm_can_support_constraints( constrained_vertices=machine_graph.partitions, supported_constraints=[], abstract_constraint_type=AbstractKeyAllocatorConstraint) # take each edge and create keys from its placement progress = ProgressBar(machine_graph.n_outgoing_edge_partitions, "Allocating routing keys") routing_infos = RoutingInfo() routing_tables = MulticastRoutingTables() for partition in progress.over(machine_graph.outgoing_edge_partitions): for edge in partition.edges: routing_infos.add_partition_info( self._allocate_partition_route(edge, placements, machine_graph, n_keys_map)) return routing_infos, routing_tables
def __call__(self, machine_graph, machine, plan_n_timesteps): """ :param MachineGraph machine_graph: The machine_graph to place :param ~spinn_machine.Machine machine: The machine with respect to which to partition the application graph :param int plan_n_timesteps: number of timesteps to plan for :return: A set of placements :rtype: Placements :raise PacmanPlaceException: If something goes wrong with the placement """ # check that the algorithm can handle the constraints self._check_constraints(machine_graph.vertices) placements = Placements() vertices = sort_vertices_by_known_constraints(machine_graph.vertices) # Iterate over vertices and generate placements progress = ProgressBar(machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph) all_vertices_placed = set() for vertex in progress.over(vertices): if vertex not in all_vertices_placed: vertices_placed = self._place_vertex(vertex, resource_tracker, machine, placements, vertices_on_same_chip, machine_graph) all_vertices_placed.update(vertices_placed) return placements
def __call__(self, app_graph): """ Add a splitter to every vertex that doesn't already have one. :param ApplicationGraph app_graph: app graph :raises PacmanConfigurationException: If a bad configuration is set """ progress_bar = ProgressBar( string_describing_what_being_progressed=self.PROGRESS_BAR_NAME, total_number_of_things_to_do=len(app_graph.vertices)) for app_vertex in progress_bar.over(app_graph.vertices): if app_vertex.splitter is None: if isinstance(app_vertex, AbstractPopulationVertex): self.abstract_pop_heuristic(app_vertex) elif isinstance(app_vertex, ApplicationSpiNNakerLinkVertex): self.external_spinnaker_link_heuristic(app_vertex) elif isinstance(app_vertex, ApplicationFPGAVertex): self.external_fpga_link_heuristic(app_vertex) elif isinstance(app_vertex, SpikeSourceArrayVertex): self.spike_source_array_heuristic(app_vertex) elif isinstance(app_vertex, SpikeSourcePoissonVertex): self.spike_source_poisson_heuristic(app_vertex) else: # go to basic selector. it might know what to do self.vertex_selector(app_vertex) if isinstance(app_vertex, AbstractAcceptsIncomingSynapses): app_vertex.verify_splitter(app_vertex.splitter)
def __call__(self, placements, hostname, report_default_directory, write_text_specs, machine, data_n_timesteps, placement_order=None): """ :param ~.Placements placements: :param str hostname: :param str report_default_directory: :param bool write_text_specs: :param ~.Machine machine: :param int data_n_timesteps: :param list(~.Placement) placement_order: :rtype: tuple(DataSpecificationTargets, dict(tuple(int,int,int), int)) :raises ConfigurationException: """ # pylint: disable=too-many-arguments, too-many-locals # pylint: disable=attribute-defined-outside-init self._machine = machine self._hostname = hostname self._report_dir = report_default_directory self._write_text = write_text_specs # iterate though vertices and call generate_data_spec for each # vertex targets = DataSpecificationTargets(machine, self._report_dir) if placement_order is None: placement_order = placements.placements progress = ProgressBar(placements.n_placements, "Generating data specifications") vertices_to_reset = list() for placement in progress.over(placement_order): # Try to generate the data spec for the placement vertex = placement.vertex generated = self.__generate_data_spec_for_vertices( placement, vertex, targets, data_n_timesteps) if generated and isinstance(vertex, AbstractRewritesDataSpecification): vertices_to_reset.append(vertex) # If the spec wasn't generated directly, and there is an # application vertex, try with that if not generated and vertex.app_vertex is not None: generated = self.__generate_data_spec_for_vertices( placement, vertex.app_vertex, targets, data_n_timesteps) if generated and isinstance(vertex.app_vertex, AbstractRewritesDataSpecification): vertices_to_reset.append(vertex.app_vertex) # Ensure that the vertices know their regions have been reloaded for vertex in vertices_to_reset: vertex.mark_regions_reloaded() return targets, self._region_sizes
def range_compressor(router_tables, accept_overflow=True): """ :param MulticastRoutingTables router_tables: :param bool accept_overflow: A flag which should only be used in testing to stop raising an exception if result is too big :rtype: MulticastRoutingTables """ if accept_overflow: message = "Precompressing tables using Range Compressor" else: message = "Compressing tables using Range Compressor" progress = ProgressBar(len(router_tables.routing_tables), message) compressor = RangeCompressor() compressed_tables = MulticastRoutingTables() for table in progress.over(router_tables.routing_tables): new_table = compressor.compress_table(table) if (new_table.number_of_entries > Machine.ROUTER_ENTRIES and not accept_overflow): raise MinimisationFailedError( f"The routing table {table.x} {table.y} with " f"{table.number_of_entries} entries after compression " f"still has {new_table.number_of_entries} so will not fit") compressed_tables.add_routing_table(new_table) logger.info(f"Ranged compressor resulted with the largest table of size " f"{compressed_tables.max_number_of_entries}") return compressed_tables
def __call__(self, machine_graph, machine, plan_n_timesteps): """ :param MachineGraph machine_graph: The machine_graph to place :param ~spinn_machine.Machine machine: A SpiNNaker machine object. :param int plan_n_timesteps: number of timesteps to plan for :return: Placements of vertices on the machine :rtype: Placements """ # check that the algorithm can handle the constraints self._check_constraints( machine_graph.vertices, additional_placement_constraints={SameChipAsConstraint}) # in order to test isomorphism include: # placements_copy = Placements() placements = Placements() vertices = sort_vertices_by_known_constraints(machine_graph.vertices) progress = ProgressBar( machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_hilbert_chips(machine)) # get vertices which must be placed on the same chip vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph) # iterate over vertices and generate placements all_vertices_placed = set() for vertex in progress.over(vertices): if vertex not in all_vertices_placed: vertices_placed = self._place_vertex( vertex, resource_tracker, machine, placements, vertices_on_same_chip) all_vertices_placed.update(vertices_placed) return placements
def __call__(self, report_default_directory, dsg_targets, transceiver): """ Creates a report that states where in SDRAM each region is \ (read from machine) :param report_default_directory: the folder where reports are written :param dsg_targets: the map between placement and file writer :param transceiver: the spinnMan instance :rtype: None """ directory_name = os.path.join(report_default_directory, MEM_MAP_SUBDIR_NAME) if not os.path.exists(directory_name): os.makedirs(directory_name) progress = ProgressBar(dsg_targets, "Writing memory map reports") for (x, y, p) in progress.over(dsg_targets): file_name = os.path.join(directory_name, MEM_MAP_FILENAME.format(x, y, p)) try: with open(file_name, "w") as f: self._describe_mem_map(f, transceiver, x, y, p) except IOError: logger.exception( "Generate_placement_reports: Can't open file" " {} for writing.", file_name)
def compress(self, register): """ Apply the on-machine compression algorithm. :param int register: number of user register to check """ # pylint: disable=too-many-arguments # build progress bar progress_bar = ProgressBar( len(self._routing_tables.routing_tables) * 2, self._progresses_text) self._compressor_app_id = self._transceiver.app_id_tracker.get_new_id() # figure size of SDRAM needed for each chip for storing the routing # table for routing_table in progress_bar.over(self._routing_tables, False): self._load_routing_table(routing_table) # load the router compressor executable executable_targets = self._load_executables() executable_finder = ExecutableFinder(binary_search_paths=[]) read_algorithm_iobuf = True run_system_application( executable_targets, self._compressor_app_id, self._transceiver, self._provenance_file_path, executable_finder, read_algorithm_iobuf, functools.partial( self._check_for_success, register=register), [CPUState.FINISHED], False, "compressor_on_{}_{}_{}.txt", [self._binary_path], progress_bar)
def __call__( self, live_packet_gatherer_parameters, placements, live_packet_gatherers_to_vertex_mapping, machine, machine_graph, application_graph=None, graph_mapper=None): """ :param live_packet_gatherer_parameters: the set of parameters :param placements: the placements object :param live_packet_gatherers_to_vertex_mapping:\ the mapping of LPG parameters and the machine vertices associated\ with it :param machine: the SpiNNaker machine :param machine_graph: the machine graph :param application_graph: the app graph :param graph_mapper: the graph mapper between app and machine graph :rtype: None """ # pylint: disable=too-many-arguments progress = ProgressBar( live_packet_gatherer_parameters, string_describing_what_being_progressed=( "Adding edges to the machine graph between the vertices to " "which live output has been requested and its local Live " "Packet Gatherer")) for lpg_params in progress.over(live_packet_gatherer_parameters): # locate vertices needed to be connected to a LPG with these params for vertex in live_packet_gatherer_parameters[lpg_params]: self._connect_lpg_vertex( application_graph, graph_mapper, machine, placements, machine_graph, vertex, live_packet_gatherers_to_vertex_mapping, lpg_params)
def _read_back_bit_fields(self, app_graph, transceiver, placements, default_report_folder, bit_field_report_name): """ report of the bitfields that were generated :param app_graph: app graph :param transceiver: the SPiNNMan instance :param placements: The placements :param default_report_folder:the file path for where reports are. :param bit_field_report_name: the name of the file :rtype: None """ # generate file progress = ProgressBar(len(app_graph.vertices), "reading back bitfields from chip") file_path = os.path.join(default_report_folder, bit_field_report_name) with open(file_path, "w") as output: # read in for each app vertex that would have a bitfield for app_vertex in progress.over(app_graph.vertices): # get machine verts for machine_vertex in app_vertex.machine_vertices: if isinstance(machine_vertex, AbstractSupportsBitFieldGeneration): self.__read_back_single_core_data( machine_vertex, placements, transceiver, output)
def generate_initial_connectivity(sigma, p, msg, N_layer=256, n=16, s_max=16, g_max=.2, delay=1.): # print "|", 256 // 4 * "-", "|" # print "|", connections = [] s = np.zeros(N_layer) pbar = ProgressBar(total_number_of_things_to_do=N_layer, string_describing_what_being_progressed=msg) for postsynaptic_neuron_index in range(N_layer): # if postsynaptic_neuron_index % 8 == 0: # print "=", pbar.update() post = (postsynaptic_neuron_index // n, postsynaptic_neuron_index % n) while s[postsynaptic_neuron_index] < s_max: potential_pre_index = np.random.randint(0, N_layer) pre = (potential_pre_index // n, potential_pre_index % n) # Commenting this 2 lines to allow for multapses # if potential_pre_index not in existing_pre[ # postsynaptic_neuron_index]: if formation_rule(pre, post, sigma, p): s[postsynaptic_neuron_index] += 1 connections.append((potential_pre_index, postsynaptic_neuron_index, g_max, delay)) # print " |" return connections
def __call__(self, machine_graph, graph_mapper): """ :param machine_graph: the machine_graph whose edges are to be filtered :param graph_mapper: the graph mapper between graphs :return: a new graph mapper and machine graph """ new_machine_graph = MachineGraph(label=machine_graph.label) new_graph_mapper = GraphMapper() # create progress bar progress = ProgressBar( machine_graph.n_vertices + machine_graph.n_outgoing_edge_partitions, "Filtering edges") # add the vertices directly, as they wont be pruned. for vertex in progress.over(machine_graph.vertices, False): self._add_vertex_to_new_graph( vertex, graph_mapper, new_machine_graph, new_graph_mapper) # start checking edges to decide which ones need pruning.... for partition in progress.over(machine_graph.outgoing_edge_partitions): for edge in partition.edges: if self._is_filterable(edge, graph_mapper): logger.debug("this edge was pruned %s", edge) continue logger.debug("this edge was not pruned %s", edge) self._add_edge_to_new_graph( edge, partition, graph_mapper, new_machine_graph, new_graph_mapper) # returned the pruned graph and graph_mapper return new_machine_graph, new_graph_mapper
def __call__(self, transceiver, machine, app_id, dsg_targets, processor_to_app_data_base_address=None): """ :param machine: the python representation of the SpiNNaker machine :param transceiver: the spinnman instance :param app_id: the application ID of the simulation :param dsg_targets: map of placement to file path :return: map of placement and DSG data, and loaded data flag. """ # pylint: disable=too-many-arguments if processor_to_app_data_base_address is None: processor_to_app_data_base_address = dict() # create a progress bar for end users progress = ProgressBar( dsg_targets, "Executing data specifications and loading data") for (x, y, p), data_spec_file_path in \ progress.over(iteritems(dsg_targets)): # write information for the memory map report processor_to_app_data_base_address[x, y, p] = self._execute( transceiver, machine, app_id, x, y, p, data_spec_file_path) return processor_to_app_data_base_address
def __call__(self, report_default_directory, machine): """ Creates a report that states where in SDRAM each region is. :param report_default_directory: the folder where reports are written :param machine: python representation of the machine :rtype: None """ # create file path directory_name = os.path.join( report_default_directory, self.AREA_CODE_REPORT_NAME) # create the progress bar for end users progress_bar = ProgressBar( len(machine.ethernet_connected_chips), "Writing the board chip report") # iterate over ethernet chips and then the chips on that board with open(directory_name, "w") as writer: for ethernet_connected_chip in \ progress_bar.over(machine.ethernet_connected_chips): chips = machine.get_chips_on_board(ethernet_connected_chip) writer.write( "board with IP address : {} : has chips {}\n".format( ethernet_connected_chip.ip_address, list(chips)))
def placement_report_without_application_graph_by_vertex( report_folder, hostname, placements, machine_graph): """ Generate report on the placement of vertices onto cores by vertex. :param str report_folder: the folder to which the reports are being written :param str hostname: the machine's hostname to which the placer worked on :param Placements placements: the placements objects built by the placer. :param MachineGraph machine_graph: the machine graph generated by the end user """ # Cycle through all vertices, and for each cycle through its vertices. # For each vertex, describe its core mapping. file_name = os.path.join(report_folder, _PLACEMENT_VTX_SIMPLE_FILENAME) time_date_string = time.strftime("%c") try: with open(file_name, "w") as f: progress = ProgressBar(machine_graph.n_vertices, "Generating placement report") f.write(" Placement Information by Vertex\n") f.write(" ===============================\n\n") f.write("Generated: {} for target machine '{}'\n\n".format( time_date_string, hostname)) for vertex in progress.over(machine_graph.vertices): _write_one_vertex_machine_placement(f, vertex, placements) except IOError: logger.exception( "Generate_placement_reports: Can't open file {} for" " writing.", file_name)
def _run(self, placements, machine_graph): """ Find routes between the edges with the allocated information, placed in the given places :param Placements placements: The placements of the edges :param ~spinn_machine.Machine machine: The machine through which the routes are to be found :param MachineGraph machine_graph: the machine_graph object :param bool use_progress_bar: whether to show a progress bar :return: The discovered routes :rtype: MulticastRoutingTableByPartition :raise PacmanRoutingException: If something goes wrong with the routing """ nodes_info = self._initiate_node_info() tables = self._initiate_dijkstra_tables() self._update_all_weights(nodes_info) # each vertex represents a core in the board progress = ProgressBar(placements.n_placements, "Creating routing entries") for placement in progress.over(placements.placements): self._route(placement, placements, machine_graph, nodes_info, tables) return self._routing_paths
def __call__(self, txrx, app_id, all_core_subsets): # check that the right number of processors are in sync processors_completed = txrx.get_core_state_count( app_id, CPUState.FINISHED) total_processors = len(all_core_subsets) left_to_do_cores = total_processors - processors_completed progress = ProgressBar( left_to_do_cores, "Forcing error cores to generate provenance data") error_cores = txrx.get_cores_in_state( all_core_subsets, CPUState.RUN_TIME_EXCEPTION) watchdog_cores = txrx.get_cores_in_state( all_core_subsets, CPUState.WATCHDOG) idle_cores = txrx.get_cores_in_state( all_core_subsets, CPUState.IDLE) if error_cores or watchdog_cores or idle_cores: raise ConfigurationException( "Some cores have crashed. RTE cores {}, watch-dogged cores {}," " idle cores {}".format( error_cores.values(), watchdog_cores.values(), idle_cores.values())) # check that all cores are in the state FINISHED which shows that # the core has received the message and done provenance updating self._update_provenance(txrx, total_processors, processors_completed, all_core_subsets, app_id, progress) progress.end()
def __call__(self, report_folder, connection_holder, dsg_targets): """ Convert synaptic matrix for every application edge. """ # Update the print options to display everything print_opts = numpy.get_printoptions() numpy.set_printoptions(threshold=numpy.nan) if dsg_targets is None: raise SynapticConfigurationException( "dsg_targets should not be none, used as a check for " "connection holder data to be generated") # generate folder for synaptic reports top_level_folder = os.path.join(report_folder, _DIRNAME) if not os.path.exists(top_level_folder): os.mkdir(top_level_folder) # create progress bar progress = ProgressBar(connection_holder.keys(), "Generating synaptic matrix reports") # for each application edge, write matrix in new file for edge, _ in progress.over(connection_holder.keys()): # only write matrix's for edges which have matrix's if isinstance(edge, ProjectionApplicationEdge): # figure new file name file_name = os.path.join( top_level_folder, _TMPL_FILENAME.format(edge.label)) self._write_file(file_name, connection_holder, edge) # Reset the print options numpy.set_printoptions(**print_opts)
def __call__(self, router_tables, target_length=None): # build storage compressed_pacman_router_tables = MulticastRoutingTables() # create progress bar progress = ProgressBar(router_tables.routing_tables, "Compressing routing Tables") # compress each router for router_table in progress.over(router_tables.routing_tables): # convert to rig format entries = self._convert_to_mundy_format(router_table) # compress the router entries compressed_router_table_entries = \ rigs_compressor.minimise(entries, target_length) # convert back to pacman model compressed_pacman_table = self._convert_to_pacman_router_table( compressed_router_table_entries, router_table.x, router_table.y) # add to new compressed routing tables compressed_pacman_router_tables.add_routing_table( compressed_pacman_table) # return return compressed_pacman_router_tables
def __call__(self, transceiver, placements, has_ran, provenance_data_objects=None): """ :param transceiver: the SpiNNMan interface object :param placements: The placements of the vertices :param has_ran: token that states that the simulation has ran """ if not has_ran: raise ConfigurationException( "This function has been called before the simulation has ran." " This is deemed an error, please rectify and try again") if provenance_data_objects is not None: prov_items = provenance_data_objects else: prov_items = list() progress = ProgressBar(placements.n_placements, "Getting provenance data") # retrieve provenance data from any cores that provide data for placement in progress.over(placements.placements): if isinstance(placement.vertex, AbstractProvidesProvenanceDataFromMachine): # get data prov_items.extend( placement.vertex.get_provenance_data_from_machine( transceiver, placement)) return prov_items
def __call__( self, placements, hostname, report_default_directory, write_text_specs, machine, graph_mapper=None, placement_order=None): """ :param placements: placements of machine graph to cores :param hostname: SpiNNaker machine name :param report_default_directory: the location where reports are stored :param write_text_specs:\ True if the textual version of the specification is to be written :param machine: the python representation of the SpiNNaker machine :param graph_mapper:\ the mapping between application and machine graph :param placement:\ the optional order in which placements should be examined :return: DSG targets (map of placement tuple and filename) """ # pylint: disable=too-many-arguments, too-many-locals # pylint: disable=attribute-defined-outside-init self._machine = machine self._hostname = hostname self._report_dir = report_default_directory self._write_text = write_text_specs # iterate though vertices and call generate_data_spec for each # vertex targets = DataSpecificationTargets(machine, self._report_dir) if placement_order is None: placement_order = placements.placements progress = ProgressBar( placements.n_placements, "Generating data specifications") vertices_to_reset = list() for placement in progress.over(placement_order): # Try to generate the data spec for the placement generated = self.__generate_data_spec_for_vertices( placement, placement.vertex, targets) if generated and isinstance( placement.vertex, AbstractRewritesDataSpecification): vertices_to_reset.append(placement.vertex) # If the spec wasn't generated directly, and there is an # application vertex, try with that if not generated and graph_mapper is not None: associated_vertex = graph_mapper.get_application_vertex( placement.vertex) generated = self.__generate_data_spec_for_vertices( placement, associated_vertex, targets) if generated and isinstance( associated_vertex, AbstractRewritesDataSpecification): vertices_to_reset.append(associated_vertex) # Ensure that the vertices know their regions have been reloaded for vertex in vertices_to_reset: vertex.mark_regions_reloaded() return targets
def __call__(self, fixed_routes, transceiver, app_id): progress_bar = ProgressBar( total_number_of_things_to_do=len(fixed_routes), string_describing_what_being_progressed="loading fixed routes") for chip_x, chip_y in progress_bar.over(fixed_routes.keys()): transceiver.load_fixed_route( chip_x, chip_y, fixed_routes[(chip_x, chip_y)], app_id)
def __call__(self, router_tables, app_id, transceiver, machine): progress = ProgressBar(router_tables.routing_tables, "Loading routing data onto the machine") # load each router table that is needed for the application to run into # the chips SDRAM for table in progress.over(router_tables.routing_tables): if (not machine.get_chip_at(table.x, table.y).virtual and table.multicast_routing_entries): transceiver.load_multicast_routes( table.x, table.y, table.multicast_routing_entries, app_id=app_id)
def __call__(self, machine_graph, machine, plan_n_timesteps): """ :param machine_graph: The machine_graph to place :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param machine:\ The machine with respect to which to partition the application\ graph :type machine: :py:class:`spinn_machine.Machine` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :return: A set of placements :rtype: :py:class:`pacman.model.placements.Placements` :raise pacman.exceptions.PacmanPlaceException: \ If something goes wrong with the placement """ # check that the algorithm can handle the constraints self._check_constraints(machine_graph.vertices) # Sort the vertices into those with and those without # placement constraints placements = Placements() constrained = list() unconstrained = set() for vertex in machine_graph.vertices: if locate_constraints_of_type( vertex.constraints, AbstractPlacerConstraint): constrained.append(vertex) else: unconstrained.add(vertex) # Iterate over constrained vertices and generate placements progress = ProgressBar( machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) constrained = sort_vertices_by_known_constraints(constrained) for vertex in progress.over(constrained, False): self._place_vertex(vertex, resource_tracker, machine, placements) while unconstrained: # Place the subgraph with the overall most connected vertex max_connected_vertex = self._find_max_connected_vertex( unconstrained, machine_graph) self._place_unconstrained_subgraph( max_connected_vertex, machine_graph, unconstrained, machine, placements, resource_tracker, progress) # finished, so stop progress bar and return placements progress.end() return placements
def __get_projection_data( self, data_to_get, pre_vertex, post_vertex, connection_holder, handle_time_out_configuration): # pylint: disable=too-many-arguments, too-many-locals ctl = self._spinnaker_control # if using extra monitor functionality, locate extra data items if ctl.get_generated_output("UsingAdvancedMonitorSupport"): extra_monitors = ctl.get_generated_output( "MemoryExtraMonitorVertices") receivers = ctl.get_generated_output( "MemoryMCGatherVertexToEthernetConnectedChipMapping") extra_monitor_placements = ctl.get_generated_output( "MemoryExtraMonitorToChipMapping") else: extra_monitors = None receivers = None extra_monitor_placements = None edges = ctl.graph_mapper.get_machine_edges(self._projection_edge) progress = ProgressBar( edges, "Getting {}s for projection between {} and {}".format( data_to_get, pre_vertex.label, post_vertex.label)) for edge in progress.over(edges): placement = ctl.placements.get_placement_of_vertex( edge.post_vertex) # if using extra monitor data extractor find local receiver if extra_monitors is not None: receiver = helpful_functions.locate_extra_monitor_mc_receiver( placement_x=placement.x, placement_y=placement.y, machine=ctl.machine, packet_gather_cores_to_ethernet_connection_map=receivers) sender_extra_monitor_core = extra_monitor_placements[ placement.x, placement.y] sender_monitor_place = ctl.placements.get_placement_of_vertex( sender_extra_monitor_core) else: receiver = None sender_monitor_place = None connections = post_vertex.get_connections_from_machine( ctl.transceiver, placement, edge, ctl.graph_mapper, ctl.routing_infos, self._synapse_information, ctl.machine_time_step, extra_monitors is not None, ctl.placements, receiver, sender_monitor_place, extra_monitors, handle_time_out_configuration, ctl.fixed_routes) if connections is not None: connection_holder.add_connections(connections) connection_holder.finish()
def __call__(self, machine_graph, placements, buffer_manager): # Count the regions to be read n_regions_to_read, recording_placements = self._count_regions( machine_graph, placements) # Read back the regions progress = ProgressBar( n_regions_to_read, "Extracting buffers from the last run") try: buffer_manager.get_data_for_placements( recording_placements, progress) finally: progress.end()
def synapse_expander( app_graph, graph_mapper, placements, transceiver, provenance_file_path, executable_finder): """ Run the synapse expander - needs to be done after data has been loaded """ synapse_expander = executable_finder.get_executable_path(SYNAPSE_EXPANDER) delay_expander = executable_finder.get_executable_path(DELAY_EXPANDER) progress = ProgressBar(len(app_graph.vertices) + 2, "Expanding Synapses") # Find the places where the synapse expander and delay receivers should run expander_cores = ExecutableTargets() for vertex in progress.over(app_graph.vertices, finish_at_end=False): # Find population vertices if isinstance( vertex, (AbstractPopulationVertex, DelayExtensionVertex)): # Add all machine vertices of the population vertex to ones # that need synapse expansion for m_vertex in graph_mapper.get_machine_vertices(vertex): vertex_slice = graph_mapper.get_slice(m_vertex) if vertex.gen_on_machine(vertex_slice): placement = placements.get_placement_of_vertex(m_vertex) if isinstance(vertex, AbstractPopulationVertex): binary = synapse_expander else: binary = delay_expander expander_cores.add_processor( binary, placement.x, placement.y, placement.p) # Launch the delay receivers expander_app_id = transceiver.app_id_tracker.get_new_id() transceiver.execute_application(expander_cores, expander_app_id) progress.update() # Wait for everything to finish finished = False try: transceiver.wait_for_cores_to_be_in_state( expander_cores.all_core_subsets, expander_app_id, [CPUState.FINISHED]) progress.update() finished = True _extract_iobuf(expander_cores, transceiver, provenance_file_path) progress.end() except Exception: logger.exception("Synapse expander has failed") _handle_failure( expander_cores, transceiver, provenance_file_path) finally: transceiver.stop_application(expander_app_id) transceiver.app_id_tracker.free_id(expander_app_id) if not finished: raise SpynnakerException( "The synapse expander failed to complete")
def __call__( self, machine, n_machine_time_steps, n_samples_per_recording, sampling_frequency, time_scale_factor, machine_time_step, pre_allocated_resources=None): """ :param pre_allocated_resources: other preallocated resources :param machine: the SpiNNaker machine as discovered :param n_machine_time_steps: the number of machine\ time steps used by the simulation during this phase :param n_samples_per_recording: how many samples between record entries :param sampling_frequency: the frequency of sampling :param time_scale_factor: the time scale factor :param machine_time_step: the machine time step :return: preallocated resources """ # pylint: disable=too-many-arguments progress_bar = ProgressBar( machine.n_chips, "Preallocating resources for chip power monitor") # store how much SDRAM the power monitor uses per core resources = ChipPowerMonitorMachineVertex.get_resources( n_machine_time_steps=n_machine_time_steps, n_samples_per_recording=n_samples_per_recording, sampling_frequency=sampling_frequency, time_scale_factor=time_scale_factor, time_step=machine_time_step) # for every Ethernet connected chip, get the resources needed by the # live packet gatherers sdrams = list() cores = list() for chip in progress_bar.over(machine.chips): sdrams.append( SpecificChipSDRAMResource(chip, resources.sdram.get_value())) cores.append(CoreResource(chip, 1)) # create preallocated resource container cpm_pre_allocated_resource_container = PreAllocatedResourceContainer( specific_sdram_usage=sdrams, core_resources=cores) # add other preallocated resources if pre_allocated_resources is not None: cpm_pre_allocated_resource_container.extend( pre_allocated_resources) # return preallocated resources return cpm_pre_allocated_resource_container
def __call__(self, app_id, txrx, executable_types): total_processors = \ len(executable_types[ExecutableType.USES_SIMULATION_INTERFACE]) all_core_subsets = \ executable_types[ExecutableType.USES_SIMULATION_INTERFACE] progress = ProgressBar( total_processors, "Turning off all the cores within the simulation") # check that the right number of processors are finished processors_finished = txrx.get_core_state_count( app_id, CPUState.FINISHED) finished_cores = processors_finished while processors_finished != total_processors: if processors_finished > finished_cores: progress.update(processors_finished - finished_cores) finished_cores = processors_finished processors_rte = txrx.get_core_state_count( app_id, CPUState.RUN_TIME_EXCEPTION) processors_watchdogged = txrx.get_core_state_count( app_id, CPUState.WATCHDOG) if processors_rte > 0 or processors_watchdogged > 0: raise ExecutableFailedToStopException( "{} of {} processors went into an error state when" " shutting down".format( processors_rte + processors_watchdogged, total_processors)) successful_cores_finished = txrx.get_cores_in_state( all_core_subsets, CPUState.FINISHED) for core_subset in all_core_subsets: for processor in core_subset.processor_ids: if not successful_cores_finished.is_core( core_subset.x, core_subset.y, processor): self._update_provenance_and_exit( txrx, processor, core_subset) time.sleep(0.5) processors_finished = txrx.get_core_state_count( app_id, CPUState.FINISHED) progress.end()
def __call__( self, placements, graph, executable_finder, graph_mapper=None): executable_targets = ExecutableTargets() progress = ProgressBar(graph.n_vertices, "Finding binaries") for vertex in progress.over(graph.vertices): placement = placements.get_placement_of_vertex(vertex) self._get_binary( placement, vertex, executable_targets, executable_finder) if graph_mapper is not None: associated_vertex = graph_mapper.get_application_vertex(vertex) self._get_binary( placement, associated_vertex, executable_targets, executable_finder) return executable_targets
def __call__(self, machine_graph=None, application_graph=None, graph_mapper=None): # Generate an n_keys map for the graph and add constraints n_keys_map = DictBasedMachinePartitionNKeysMap() if machine_graph is None: raise ConfigurationException( "A machine graph is required for this mapper. " "Please choose and try again") if (application_graph is None) != (graph_mapper is None): raise ConfigurationException( "Can only do one graph. semantically doing 2 graphs makes no " "sense. Please choose and try again") if application_graph is not None: # generate progress bar progress = ProgressBar( machine_graph.n_vertices, "Getting number of keys required by each edge using " "application graph") # iterate over each partition in the graph for vertex in progress.over(machine_graph.vertices): partitions = machine_graph.\ get_outgoing_edge_partitions_starting_at_vertex( vertex) for partition in partitions: if partition.traffic_type == EdgeTrafficType.MULTICAST: self._process_application_partition( partition, n_keys_map, graph_mapper) else: # generate progress bar progress = ProgressBar( machine_graph.n_vertices, "Getting number of keys required by each edge using " "machine graph") for vertex in progress.over(machine_graph.vertices): partitions = machine_graph.\ get_outgoing_edge_partitions_starting_at_vertex( vertex) for partition in partitions: if partition.traffic_type == EdgeTrafficType.MULTICAST: self._process_machine_partition(partition, n_keys_map) return n_keys_map
def __call__( self, routing_infos, routing_table_by_partitions, machine): """ :param routing_infos: :param routing_table_by_partitions: :param machine: """ progress = ProgressBar(machine.n_chips, "Generating routing tables") routing_tables = MulticastRoutingTables() for chip in progress.over(machine.chips): partitions_in_table = routing_table_by_partitions.\ get_entries_for_router(chip.x, chip.y) if partitions_in_table: routing_tables.add_routing_table(self._create_routing_table( chip, partitions_in_table, routing_infos)) return routing_tables
def __call__( self, placements, tags, txrx, uses_advanced_monitors, report_folder, extra_monitor_cores=None, extra_monitor_to_chip_mapping=None, packet_gather_cores_to_ethernet_connection_map=None, machine=None, fixed_routes=None, java_caller=None): """ :param placements: :param tags: :param txrx: :param uses_advanced_monitors: :param extra_monitor_cores: :param extra_monitor_to_chip_mapping: :param packet_gather_cores_to_ethernet_connection_map: :param machine: :param fixed_routes: :param report_folder: The path where the SQLite database holding the data will be placed, and where any java provenance can be written. :type report_folder: str :return: """ # pylint: disable=too-many-arguments progress = ProgressBar(placements.n_placements, "Initialising buffers") # Create the buffer manager buffer_manager = BufferManager( placements=placements, tags=tags, transceiver=txrx, extra_monitor_cores=extra_monitor_cores, packet_gather_cores_to_ethernet_connection_map=( packet_gather_cores_to_ethernet_connection_map), extra_monitor_to_chip_mapping=extra_monitor_to_chip_mapping, machine=machine, uses_advanced_monitors=uses_advanced_monitors, fixed_routes=fixed_routes, report_folder=report_folder, java_caller=java_caller) for placement in progress.over(placements.placements): if isinstance(placement.vertex, AbstractSendsBuffersFromHost): if placement.vertex.buffering_input(): buffer_manager.add_sender_vertex(placement.vertex) if isinstance(placement.vertex, AbstractReceiveBuffersToHost): buffer_manager.add_receiving_vertex(placement.vertex) return buffer_manager
def generate_machine_edges(machine_graph, graph_mapper, application_graph): """ Generate the machine edges for the vertices in the graph :param machine_graph: the machine graph to add edges to :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param graph_mapper: the mapper graphs :type graph_mapper:\ :py:class:`pacman.model.graphs.common.GraphMapper` :param application_graph: the application graph to work with :type application_graph:\ :py:class:`pacman.model.graphs.application.ApplicationGraph` """ # start progress bar progress = ProgressBar( machine_graph.n_vertices, "Partitioning graph edges") # Partition edges according to vertex partitioning for source_vertex in progress.over(machine_graph.vertices): # For each out edge of the parent vertex... vertex = graph_mapper.get_application_vertex(source_vertex) application_outgoing_partitions = application_graph.\ get_outgoing_edge_partitions_starting_at_vertex(vertex) for application_partition in application_outgoing_partitions: for application_edge in application_partition.edges: # create new partitions for dest_vertex in graph_mapper.get_machine_vertices( application_edge.post_vertex): machine_edge = application_edge.create_machine_edge( source_vertex, dest_vertex, "machine_edge_for{}".format(application_edge.label)) machine_graph.add_edge( machine_edge, application_partition.identifier) # add constraints from the application partition machine_partition = machine_graph.\ get_outgoing_edge_partition_starting_at_vertex( source_vertex, application_partition.identifier) machine_partition.add_constraints( application_partition.constraints) # update mapping object graph_mapper.add_edge_mapping( machine_edge, application_edge)
def __call__(self, machine_graph, machine, file_path): """ :param machine_graph: the machine graph :param machine: the machine """ progress = ProgressBar( machine_graph.n_vertices + 2, "creating JSON constraints") json_obj = list() self._add_monitor_core_reserve(json_obj) progress.update() self._add_extra_monitor_cores(json_obj, machine) progress.update() vertex_by_id = self._search_graph_for_placement_constraints( json_obj, machine_graph, machine, progress) with open(file_path, "w") as f: json.dump(json_obj, f) # validate the schema file_format_schemas.validate(json_obj, "constraints.json") # complete progress bar progress.end() return file_path, vertex_by_id
def allocate_chip_ids(self, machine, graph): """ Go through the chips (real and virtual) and allocate keys for each """ progress = ProgressBar( graph.n_vertices + machine.n_chips, "Allocating virtual identifiers") # allocate standard IDs for real chips for x, y in progress.over(machine.chip_coordinates, False): expected_chip_id = (x << 8) + y self._allocate_elements(expected_chip_id, 1) # allocate IDs for virtual chips for vertex in progress.over(graph.vertices): if isinstance(vertex, AbstractVirtualVertex): x, y = self._assign_virtual_chip_info( machine, self._get_link_data(machine, vertex)) vertex.set_virtual_chip_coordinates(x, y)