def get_v(self, label, buffer_manager, region, state_region, placements, graph_mapper, partitionable_vertex): subvertices = \ graph_mapper.get_subvertices_from_vertex(partitionable_vertex) ms_per_tick = self._machine_time_step / 1000.0 data = list() missing_str = "" progress_bar = \ ProgressBar(len(subvertices), "Getting membrane voltage for {}".format(label)) for subvertex in subvertices: vertex_slice = graph_mapper.get_subvertex_slice(subvertex) placement = placements.get_placement_of_subvertex(subvertex) x = placement.x y = placement.y p = placement.p # for buffering output info is taken form the buffer manager neuron_param_region_data_pointer, missing_data =\ buffer_manager.get_data_for_vertex( placement, region, state_region) if missing_data: missing_str += "({}, {}, {}); ".format(x, y, p) record_raw = neuron_param_region_data_pointer.read_all() record_length = len(record_raw) n_rows = record_length / ((vertex_slice.n_atoms + 1) * 4) record = (numpy.asarray(record_raw, dtype="uint8"). view(dtype="<i4")).reshape((n_rows, (vertex_slice.n_atoms + 1))) split_record = numpy.array_split(record, [1, 1], 1) record_time = numpy.repeat( split_record[0] * float(ms_per_tick), vertex_slice.n_atoms, 1) record_ids = numpy.tile( numpy.arange(vertex_slice.lo_atom, vertex_slice.hi_atom + 1), len(record_time)).reshape((-1, vertex_slice.n_atoms)) record_membrane_potential = split_record[2] / 32767.0 part_data = numpy.dstack( [record_ids, record_time, record_membrane_potential]) part_data = numpy.reshape(part_data, [-1, 3]) data.append(part_data) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing membrane voltage data in region {}" " from the following cores: {}".format( label, region, missing_str)) data = numpy.vstack(data) order = numpy.lexsort((data[:, 1], data[:, 0])) result = data[order] return result
def __call__(self, report_folder, application_graph): """ :param report_folder: the report folder to put figure into :param application_graph: the app graph :rtype: None """ try: import graphviz except: raise SpynnakerException( "graphviz is required to use this report. Please install" " graphviz if you want to use this report.") # create holders for data vertex_holders = dict() dot_diagram = graphviz.Digraph( comment="The graph of the network in graphical form") # build progress bar for the vertices, edges, and rendering progress_bar = ProgressBar( application_graph.n_vertices + application_graph.n_outgoing_edge_partitions + 1, "generating the graphical representation of the neural network") # write vertices into dot diagram vertex_counter = 0 for vertex in application_graph.vertices: dot_diagram.node( "{}".format(vertex_counter), "{} ({} neurons)".format(vertex.label, vertex.n_atoms)) vertex_holders[vertex] = vertex_counter vertex_counter += 1 progress_bar.update() # write edges into dot diagram for partition in application_graph.outgoing_edge_partitions: for edge in partition.edges: source_vertex_id = vertex_holders[edge.pre_vertex] dest_vertex_id = vertex_holders[edge.post_vertex] if isinstance(edge, ProjectionApplicationEdge): for synapse_info in edge.synapse_information: dot_diagram.edge( "{}".format(source_vertex_id), "{}".format(dest_vertex_id), "{}".format(synapse_info.connector)) else: dot_diagram.edge( "{}".format(source_vertex_id), "{}".format(dest_vertex_id)) progress_bar.update() # write dot file and generate pdf file_to_output = os.path.join(report_folder, "network_graph.gv") dot_diagram.render(file_to_output, view=False) progress_bar.update() progress_bar.end()
def get_spikes(self, label, buffer_manager, region, state_region, placements, graph_mapper, partitionable_vertex, base_key_function): results = list() missing_str = "" ms_per_tick = self._machine_time_step / 1000.0 subvertices = \ graph_mapper.get_subvertices_from_vertex(partitionable_vertex) progress_bar = ProgressBar(len(subvertices), "Getting spikes for {}".format(label)) for subvertex in subvertices: placement = placements.get_placement_of_subvertex(subvertex) subvertex_slice = graph_mapper.get_subvertex_slice(subvertex) x = placement.x y = placement.y p = placement.p # Read the spikes raw_spike_data, data_missing = \ buffer_manager.get_data_for_vertex( placement, region, state_region) if data_missing: missing_str += "({}, {}, {}); ".format(x, y, p) spike_data = str(raw_spike_data.read_all()) number_of_bytes_written = len(spike_data) offset = 0 while offset < number_of_bytes_written: eieio_header = EIEIODataHeader.from_bytestring( spike_data, offset) offset += eieio_header.size timestamp = eieio_header.payload_base * ms_per_tick timestamps = numpy.repeat([timestamp], eieio_header.count) keys = numpy.frombuffer( spike_data, dtype="<u4", count=eieio_header.count, offset=offset) neuron_ids = ((keys - base_key_function(subvertex)) + subvertex_slice.lo_atom) offset += eieio_header.count * 4 results.append(numpy.dstack((neuron_ids, timestamps))[0]) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing spike data in region {} from the" " following cores: {}".format(label, region, missing_str)) if len(results) != 0: result = numpy.vstack(results) result = result[numpy.lexsort((result[:, 1], result[:, 0]))] else: result = [] return result
def update_runtime(self, run_time, infinite_run, core_subsets, n_cores): self._progress_bar = ProgressBar(n_cores, "Updating run time") for core_subset in core_subsets: for processor_id in core_subset.processor_ids: self._send_request( SCPUpdateRuntimeRequest( core_subset.x, core_subset.y, processor_id, run_time, infinite_run, constants.SDP_PORTS.RUNNING_COMMAND_SDP_PORT.value), callback=self.receive_response) self._finish() self._progress_bar.end() self.check_for_error()
def load_initial_buffers(self): """ Load the initial buffers for the senders using mem writes """ total_data = 0 for vertex in self._sender_vertices: for region in vertex.get_regions(): total_data += vertex.get_region_buffer_size(region) progress_bar = ProgressBar( total_data, "Loading buffers ({} bytes)".format(total_data)) for vertex in self._sender_vertices: for region in vertex.get_regions(): self._send_initial_messages(vertex, region, progress_bar) progress_bar.end()
def _handle_external_algorithm(self, algorithm): """ Creates the input files for the algorithm :param algorithm: the algorithm :return: None """ input_params = self._create_input_commands(algorithm) inputs = \ [a.format(**input_params) for a in algorithm.command_line_args] # output debug info in case things go wrong logger.debug( "The inputs to the external mapping function are {}".format( inputs)) # create progress bar for external algorithm algorithm_progress_bar = ProgressBar( 1, "Running external algorithm {}".format(algorithm.algorithm_id)) timer = None if self._do_timing: timer = Timer() timer.start_timing() # execute other command child = subprocess.Popen(inputs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) child.wait() algorithm_progress_bar.end() if self._do_timing: self._update_timings(timer, algorithm) # check the return code for a successful execution if child.returncode != 0: stdout, stderr = child.communicate() raise exceptions.\ PacmanExternalAlgorithmFailedToCompleteException( "Algorithm {} returned a non-zero error code {}\n" " Inputs: {}\n" " Output: {}\n" " Error: {}\n".format( algorithm.algorithm_id, child.returncode, inputs, stdout, stderr)) outputs = self._sort_out_external_algorithm_outputs(algorithm) self._map_output_parameters(outputs, algorithm)
def tag_allocator_report(report_folder, tag_infos): """ Reports the tags that are being used by the tool chain for this\ simulation :param report_folder: the folder to which the reports are being written :param tag_infos: the tags container generated by the tools. :return: None """ progress_bar = ProgressBar( len(list(tag_infos.ip_tags)) + len(list(tag_infos.reverse_ip_tags)), "Reporting Tags") file_name = os.path.join(report_folder, "tags.rpt") f_routing = None try: f_routing = open(file_name, "w") except IOError: logger.error("Generate_tag_report: Can't open file {} for " "writing.".format(file_name)) for ip_tag in tag_infos.ip_tags: f_routing.write("{}".format(ip_tag)) progress_bar.update() for reverse_ip_tag in tag_infos.reverse_ip_tags: f_routing.write("{}".format(reverse_ip_tag)) progress_bar.update() f_routing.flush() f_routing.close() progress_bar.end()
def __call__(self, partitioned_graph, partitionable_graph=None, graph_mapper=None): # Generate an n_keys map for the graph and add constraints n_keys_map = DictBasedPartitionedPartitionNKeysMap() # generate progress bar progress_bar = ProgressBar( len(partitioned_graph.subvertices), "Deducing edge to number of keys map") # contains a partitionable vertex if partitionable_graph is not None and graph_mapper is not None: # iterate over each partition in the partitioned graph for vertex in partitioned_graph.subvertices: partitions = \ partitioned_graph.outgoing_edges_partitions_from_vertex( vertex) for partition_id in partitions: partition = partitions[partition_id] added_constraints = False constraints = self._process_partitionable_partition( partition, n_keys_map, partition_id, graph_mapper, partitionable_graph) if not added_constraints: partition.add_constraints(constraints) else: self._check_constraints_equal( constraints, partition.constraints) progress_bar.update() progress_bar.end() else: for vertex in partitioned_graph.subvertices: partitions = \ partitioned_graph.outgoing_edges_partitions_from_vertex( vertex) for partition_id in partitions: partition = partitions[partition_id] added_constraints = False constraints = self._process_partitioned_partition( partition, n_keys_map, partition_id, partitioned_graph) if not added_constraints: partition.add_constraints(constraints) else: self._check_constraints_equal( constraints, partition.constraints) progress_bar.update() progress_bar.end() return {'n_keys_map': n_keys_map}
def _get_synaptic_data(self, as_list, data_to_get): post_vertex = self._projection_edge.post_vertex pre_vertex = self._projection_edge.pre_vertex # If in virtual board mode, the connection data should be set if self._virtual_connection_list is not None: post_vertex = self._projection_edge.post_vertex pre_vertex = self._projection_edge.pre_vertex return ConnectionHolder( data_to_get, as_list, pre_vertex.n_atoms, post_vertex.n_atoms, self._virtual_connection_list) connection_holder = ConnectionHolder( data_to_get, as_list, pre_vertex.n_atoms, post_vertex.n_atoms) # If we haven't run, add the holder to get connections, and return it if not self._spinnaker.has_ran: post_vertex.add_pre_run_connection_holder( connection_holder, self._projection_edge, self._synapse_information) return connection_holder # Otherwise, get the connections now graph_mapper = self._spinnaker.graph_mapper placements = self._spinnaker.placements transceiver = self._spinnaker.transceiver routing_infos = self._spinnaker.routing_infos partitioned_graph = self._spinnaker.partitioned_graph subedges = graph_mapper.get_partitioned_edges_from_partitionable_edge( self._projection_edge) progress = ProgressBar( len(subedges), "Getting {}s for projection between {} and {}".format( data_to_get, pre_vertex.label, post_vertex.label)) for subedge in subedges: placement = placements.get_placement_of_subvertex( subedge.post_subvertex) connections = post_vertex.get_connections_from_machine( transceiver, placement, subedge, graph_mapper, routing_infos, self._synapse_information, partitioned_graph) if connections is not None: connection_holder.add_connections(connections) progress.update() progress.end() connection_holder.finish() return connection_holder
def __call__(self, subgraph, graph_mapper): """ :param subgraph: the subgraph whose edges are to be filtered :param graph_mapper: the graph mapper between partitionable and \ partitioned graphs. :return: a new graph mapper and partitioned graph """ new_sub_graph = PartitionedGraph(label=subgraph.label) new_graph_mapper = GraphMapper(graph_mapper.first_graph_label, subgraph.label) # create progress bar progress_bar = ProgressBar( len(subgraph.subvertices) + len(subgraph.subedges), "Filtering edges") # add the subverts directly, as they wont be pruned. for subvert in subgraph.subvertices: new_sub_graph.add_subvertex(subvert) associated_vertex = graph_mapper.get_vertex_from_subvertex(subvert) vertex_slice = graph_mapper.get_subvertex_slice(subvert) new_graph_mapper.add_subvertex(subvertex=subvert, vertex_slice=vertex_slice, vertex=associated_vertex) progress_bar.update() # start checking subedges to decide which ones need pruning.... for subvert in subgraph.subvertices: out_going_partitions = \ subgraph.outgoing_edges_partitions_from_vertex(subvert) for partitioner_identifier in out_going_partitions: for subedge in \ out_going_partitions[partitioner_identifier].edges: if not self._is_filterable(subedge, graph_mapper): logger.debug( "this subedge was not pruned {}".format(subedge)) new_sub_graph.add_subedge(subedge, partitioner_identifier) associated_edge = graph_mapper.\ get_partitionable_edge_from_partitioned_edge( subedge) new_graph_mapper.add_partitioned_edge( subedge, associated_edge) else: logger.debug( "this subedge was pruned {}".format(subedge)) progress_bar.update() progress_bar.end() # returned the pruned partitioned_graph and graph_mapper return { 'new_sub_graph': new_sub_graph, 'new_graph_mapper': new_graph_mapper }
def __call__(self, router_tables): tables = MulticastRoutingTables() previous_masks = dict() progress = ProgressBar( len(router_tables.routing_tables) * 2, "Compressing Routing Tables") # Create all masks without holes allowed_masks = [0xFFFFFFFFL - ((2**i) - 1) for i in range(33)]
def __call__(self, subgraph, graph_mapper): """ :param subgraph: the subgraph whose edges are to be updated :param graph_mapper: the graph mapper between partitionable and \ partitioned graphs. """ # create progress bar progress_bar = ProgressBar( len(subgraph.subedges), "Updating edge weights") # start checking subedges to decide which ones need pruning.... for subedge in subgraph.subedges: if isinstance(subedge, AbstractWeightUpdatable): subedge.update_weight(graph_mapper) progress_bar.update() progress_bar.end() # return nothing return {'subgraph': subgraph}
def __call__(self, router_tables, target_length=None): # build storage compressed_pacman_router_tables = MulticastRoutingTables() # create progress bar progress_bar = ProgressBar( len(router_tables.routing_tables), "Compressing routing Tables") # compress each router for router_table in router_tables.routing_tables: # convert to rig format entries = self._convert_to_mundy_format(router_table) # compress the router entries compressed_router_table_entries = \ rigs_compressor.minimise(entries, target_length) # convert back to pacman model compressed_pacman_table = self._convert_to_pacman_router_table( compressed_router_table_entries, router_table.x, router_table.y) # add to new compressed routing tables compressed_pacman_router_tables.add_routing_table( compressed_pacman_table) progress_bar.update() progress_bar.end() # return return {'routing_tables': compressed_pacman_router_tables}
def __call__(self, partitioned_graph, machine): """ :param partitioned_graph: The partitioned_graph to measure :type partitioned_graph:\ :py:class:`pacman.model.partitioned_graph.partitioned_graph.PartitionedGraph` :return: The size of the graph in number of chips :rtype: int """ # check that the algorithm can handle the constraints utility_calls.check_algorithm_can_support_constraints( constrained_vertices=partitioned_graph.subvertices, supported_constraints=[PlacerChipAndCoreConstraint], abstract_constraint_type=AbstractPlacerConstraint) ordered_subverts = utility_calls.sort_objects_by_constraint_authority( partitioned_graph.subvertices) # Iterate over subvertices and allocate progress_bar = ProgressBar(len(ordered_subverts), "Measuring the partitioned graph") resource_tracker = ResourceTracker(machine) for subvertex in ordered_subverts: resource_tracker.allocate_constrained_resources( subvertex.resources_required, subvertex.constraints) progress_bar.update() progress_bar.end() return {'n_chips': len(resource_tracker.keys)}
def __call__(self, routing_infos, routing_table_by_partitions, machine): """ :param routing_infos: :param routing_table_by_partitions: :param machine: :return: """ progress_bar = ProgressBar(len(list(machine.chips)), "Generating routing tables") routing_tables = MulticastRoutingTables() for chip in machine.chips: partitions_in_table = routing_table_by_partitions.\ get_entries_for_router(chip.x, chip.y) if len(partitions_in_table) != 0: routing_table = MulticastRoutingTable(chip.x, chip.y) for partition in partitions_in_table: keys_and_masks = routing_infos.\ get_keys_and_masks_from_partition(partition) entry = partitions_in_table[partition] for key_and_mask in keys_and_masks: multicast_routing_entry = MulticastRoutingEntry( routing_entry_key=key_and_mask.key_combo, defaultable=entry.defaultable, mask=key_and_mask.mask, link_ids=entry.out_going_links, processor_ids=entry.out_going_processors) routing_table.add_mutlicast_routing_entry( multicast_routing_entry) routing_tables.add_routing_table(routing_table) progress_bar.update() progress_bar.end() return {"router_tables": routing_tables}
def __call__(self, partitioned_graph, machine): # check that the algorithm can handle the constraints utility_calls.check_algorithm_can_support_constraints( constrained_vertices=partitioned_graph.subvertices, supported_constraints=[ PlacerRadialPlacementFromChipConstraint, TagAllocatorRequireIptagConstraint, TagAllocatorRequireReverseIptagConstraint, PlacerChipAndCoreConstraint], abstract_constraint_type=AbstractPlacerConstraint) placements = Placements() ordered_subverts = utility_calls.sort_objects_by_constraint_authority( partitioned_graph.subvertices) # Iterate over subvertices and generate placements progress_bar = ProgressBar(len(ordered_subverts), "Placing graph vertices") resource_tracker = ResourceTracker( machine, self._generate_radial_chips(machine)) for vertex in ordered_subverts: self._place_vertex(vertex, resource_tracker, machine, placements) progress_bar.update() progress_bar.end() return {'placements': placements}
def routing_info_report(report_folder, partitioned_graph, routing_infos): """ Generates a report which says which keys is being allocated to each\ subvertex :param report_folder: the report folder to store this value :param partitioned_graph: :param routing_infos: """ file_name = os.path.join(report_folder, "virtual_key_space_information_report.rpt") output = None try: output = open(file_name, "w") except IOError: logger.error("generate virtual key space information report: " "Can't open file {} for writing.".format(file_name)) progress_bar = ProgressBar(len(partitioned_graph.subvertices), "Generating Routing info report") for subvert in partitioned_graph.subvertices: output.write("Subvert: {} \n".format(subvert)) partitions = \ partitioned_graph.outgoing_edges_partitions_from_vertex(subvert) for partition in partitions.values(): keys_and_masks = \ routing_infos.get_keys_and_masks_from_partition(partition) for subedge in partition.edges: output.write("subedge:{}, keys_and_masks:{} \n".format( subedge, keys_and_masks)) output.write("\n\n") progress_bar.update() progress_bar.end() output.flush() output.close()
def __call__(self, application_graph): """ :param application_graph: app graph :return: the set of connection holders for after dsg generation """ progress_bar = ProgressBar( application_graph.n_outgoing_edge_partitions, "Generating connection holders for reporting connection data.") data_holders = dict() for partition in application_graph.outgoing_edge_partitions: for edge in partition.edges: # add pre run generators so that reports can extract without # going to machine. if isinstance(edge, ProjectionApplicationEdge): # build connection holders connection_holder = ConnectionHolder( None, True, edge.pre_vertex.n_atoms, edge.post_vertex.n_atoms) for synapse_information in edge.synapse_information: edge.post_vertex.add_pre_run_connection_holder( connection_holder, edge, synapse_information) # store for the report generations data_holders[(edge, synapse_information)] = \ connection_holder progress_bar.update() progress_bar.end() # return the two holders return data_holders
def __call__(self, partitioned_graph, placements, buffer_manager, ran_token): if not ran_token: raise exceptions.ConfigurationException( "The ran token has not been set") # Count the regions to be read n_regions_to_read = 0 for vertex in partitioned_graph.subvertices: if isinstance(vertex, AbstractReceiveBuffersToHost): n_regions_to_read += len(vertex.get_buffered_regions()) progress_bar = ProgressBar(n_regions_to_read, "Extracting buffers from the last run") # Read back the regions for vertex in partitioned_graph.subvertices: if isinstance(vertex, AbstractReceiveBuffersToHost): placement = placements.get_placement_of_subvertex(vertex) state_region = vertex.get_buffered_state_region() for region in vertex.get_buffered_regions(): buffer_manager.get_data_for_vertex(placement, region, state_region) progress_bar.update() progress_bar.end()
def get_v(self, label, buffer_manager, region, state_region, placements, graph_mapper, partitionable_vertex): subvertices = \ graph_mapper.get_subvertices_from_vertex(partitionable_vertex) ms_per_tick = self._machine_time_step / 1000.0 data = list() missing_str = "" progress_bar = \ ProgressBar(len(subvertices), "Getting membrane voltage for {}".format(label)) for subvertex in subvertices: vertex_slice = graph_mapper.get_subvertex_slice(subvertex) placement = placements.get_placement_of_subvertex(subvertex) x = placement.x y = placement.y p = placement.p # for buffering output info is taken form the buffer manager neuron_param_region_data_pointer, missing_data =\ buffer_manager.get_data_for_vertex( placement, region, state_region) if missing_data: missing_str += "({}, {}, {}); ".format(x, y, p) record_raw = neuron_param_region_data_pointer.read_all() record_length = len(record_raw) n_rows = record_length / ((vertex_slice.n_atoms + 1) * 4) record = (numpy.asarray(record_raw, dtype="uint8").view(dtype="<i4")).reshape( (n_rows, (vertex_slice.n_atoms + 1))) split_record = numpy.array_split(record, [1, 1], 1) record_time = numpy.repeat(split_record[0] * float(ms_per_tick), vertex_slice.n_atoms, 1) record_ids = numpy.tile( numpy.arange(vertex_slice.lo_atom, vertex_slice.hi_atom + 1), len(record_time)).reshape((-1, vertex_slice.n_atoms)) record_membrane_potential = split_record[2] / 32767.0 part_data = numpy.dstack( [record_ids, record_time, record_membrane_potential]) part_data = numpy.reshape(part_data, [-1, 3]) data.append(part_data) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing membrane voltage data in region {}" " from the following cores: {}".format(label, region, missing_str)) data = numpy.vstack(data) order = numpy.lexsort((data[:, 1], data[:, 0])) result = data[order] return result
def __call__(self, machine_graph, graph_mapper): """ :param machine_graph: the machine_graph whose edges are to be filtered :param graph_mapper: the graph mapper between graphs :return: a new graph mapper and machine graph """ new_machine_graph = MachineGraph(label=machine_graph.label) new_graph_mapper = GraphMapper() # create progress bar progress_bar = ProgressBar( machine_graph.n_vertices + machine_graph.n_outgoing_edge_partitions, "Filtering edges") # add the vertices directly, as they wont be pruned. for vertex in machine_graph.vertices: new_machine_graph.add_vertex(vertex) associated_vertex = graph_mapper.get_application_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) new_graph_mapper.add_vertex_mapping( machine_vertex=vertex, vertex_slice=vertex_slice, application_vertex=associated_vertex) progress_bar.update() # start checking edges to decide which ones need pruning.... for partition in machine_graph.outgoing_edge_partitions: for edge in partition.edges: if not self._is_filterable(edge, graph_mapper): logger.debug("this edge was not pruned {}".format(edge)) new_machine_graph.add_edge(edge, partition.identifier) app_edge = graph_mapper.get_application_edge(edge) new_graph_mapper.add_edge_mapping(edge, app_edge) # add partition constraints from the original graph to # the new graph # add constraints from the application partition new_machine_graph_partition = new_machine_graph.\ get_outgoing_edge_partition_starting_at_vertex( edge.pre_vertex, partition.identifier) new_machine_graph_partition.add_constraints( partition.constraints) else: logger.debug("this edge was pruned {}".format(edge)) progress_bar.update() progress_bar.end() # returned the pruned graph and graph_mapper return new_machine_graph, new_graph_mapper
def _write_router_provenance_data(self, router_tables, machine, txrx): """ Writes the provenance data of the router diagnostics :param router_tables: the routing tables generated by pacman :param machine: the spinnMachine object :param txrx: the transceiver object :return: None """ progress = ProgressBar(machine.n_chips, "Getting Router Provenance") # acquire diagnostic data items = list() seen_chips = set() for router_table in sorted( router_tables.routing_tables, key=lambda table: (table.x, table.y)): x = router_table.x y = router_table.y if not machine.get_chip_at(x, y).virtual: router_diagnostic = txrx.get_router_diagnostics(x, y) seen_chips.add((x, y)) reinjector_status = txrx.get_reinjection_status(x, y) items.extend(self._write_router_diagnostics( x, y, router_diagnostic, reinjector_status, True)) self._add_totals(router_diagnostic, reinjector_status) progress.update() for chip in sorted(machine.chips, key=lambda c: (c.x, c.y)): if not chip.virtual and (chip.x, chip.y) not in seen_chips: try: diagnostic = txrx.get_router_diagnostics(chip.x, chip.y) if (diagnostic.n_dropped_multicast_packets != 0 or diagnostic.n_local_multicast_packets != 0 or diagnostic.n_external_multicast_packets != 0): reinjector_status = txrx.get_reinjection_status( chip.x, chip.y) items.extend(self._write_router_diagnostics( chip.x, chip.y, diagnostic, reinjector_status, False)) self._add_totals(diagnostic, reinjector_status) progress.update() except Exception: # There could be issues with unused chips - don't worry! pass progress.end() return items
class UpdateRuntimeProcess(AbstractMultiConnectionProcess): def __init__(self, connection_selector): AbstractMultiConnectionProcess.__init__(self, connection_selector) self._progress_bar = None def receive_response(self, response): if self._progress_bar is not None: self._progress_bar.update() def update_runtime(self, run_time, infinite_run, core_subsets, n_cores): self._progress_bar = ProgressBar(n_cores, "Updating run time") for core_subset in core_subsets: for processor_id in core_subset.processor_ids: self._send_request( SCPUpdateRuntimeRequest( core_subset.x, core_subset.y, processor_id, run_time, infinite_run, constants.SDP_PORTS.RUNNING_COMMAND_SDP_PORT.value), callback=self.receive_response) self._finish() self._progress_bar.end() self.check_for_error()
def __call__(self, report_folder, connection_holder, dsg_targets): """ converts synaptic matrix for every application edge. """ # Update the print options to display everything import numpy print_opts = numpy.get_printoptions() numpy.set_printoptions(threshold=numpy.nan) if dsg_targets is None: raise SynapticConfigurationException( "dsg targets should not be none, used as a check for " "connection holder data to be generated") # generate folder for synaptic reports top_level_folder = os.path.join(report_folder, "synaptic_matrix_reports") if not os.path.exists(top_level_folder): os.mkdir(top_level_folder) # create progress bar progress = ProgressBar(len(connection_holder.keys()), "Generating synaptic matrix reports") # for each application edge, write matrix in new file for application_edge, _ in connection_holder.keys(): # only write matrix's for edges which have matrix's if isinstance(application_edge, ProjectionApplicationEdge): # figure new file name file_name = os.path.join( top_level_folder, "synaptic_matrix_for_application_edge_{}".format( application_edge.label)) # open writer output = None try: output = open(file_name, "w") except IOError: logger.error("Generate_placement_reports: Can't open file" " {} for writing.".format(file_name)) # write all data for all synapse_information's in same file for info in application_edge.synapse_information: this_connection_holder = connection_holder[( application_edge, info)] output.write("{}".format(this_connection_holder)) output.flush() output.close() progress.update() progress.end() # Reset the print options numpy.set_printoptions(**print_opts)
def get_spikes(self, label, buffer_manager, region, placements, graph_mapper, application_vertex, base_key_function, machine_time_step): results = list() missing_str = "" ms_per_tick = machine_time_step / 1000.0 vertices = \ graph_mapper.get_machine_vertices(application_vertex) progress_bar = ProgressBar(len(vertices), "Getting spikes for {}".format(label)) for vertex in vertices: placement = placements.get_placement_of_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) x = placement.x y = placement.y p = placement.p # Read the spikes raw_spike_data, data_missing = \ buffer_manager.get_data_for_vertex(placement, region) if data_missing: missing_str += "({}, {}, {}); ".format(x, y, p) spike_data = str(raw_spike_data.read_all()) number_of_bytes_written = len(spike_data) offset = 0 while offset < number_of_bytes_written: eieio_header = EIEIODataHeader.from_bytestring( spike_data, offset) offset += eieio_header.size timestamp = eieio_header.payload_base * ms_per_tick timestamps = numpy.repeat([timestamp], eieio_header.count) keys = numpy.frombuffer(spike_data, dtype="<u4", count=eieio_header.count, offset=offset) neuron_ids = ((keys - base_key_function(vertex)) + vertex_slice.lo_atom) offset += eieio_header.count * 4 results.append(numpy.dstack((neuron_ids, timestamps))[0]) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing spike data in region {} from the" " following cores: {}".format(label, region, missing_str)) if len(results) != 0: result = numpy.vstack(results) result = result[numpy.lexsort((result[:, 1], result[:, 0]))] else: result = [] return result
def router_report_from_paths( report_folder, routing_tables, routing_infos, hostname, partitioned_graph, placements, machine): """ Generates a text file of routing paths :param routing_tables: :param report_folder: :param hostname: :param routing_infos: :param partitioned_graph: :param placements: :param machine: :return: """ file_name = os.path.join(report_folder, "edge_routing_info.rpt") f_routing = None try: f_routing = open(file_name, "w") except IOError: logger.error("Generate_routing_reports: Can't open file {} for " "writing.".format(file_name)) f_routing.write(" Edge Routing Report\n") f_routing.write(" ===================\n\n") time_date_string = time.strftime("%c") f_routing.write("Generated: {}".format(time_date_string)) f_routing.write(" for target machine '{}'".format(hostname)) f_routing.write("\n\n") progress_bar = ProgressBar(len(partitioned_graph.subedges), "Generating Routing path report") for edge in partitioned_graph.subedges: source_placement = placements.get_placement_of_subvertex( edge.pre_subvertex) destination_placement = placements.get_placement_of_subvertex( edge.post_subvertex) partition = partitioned_graph.get_partition_of_subedge(edge) key_and_mask = routing_infos.get_keys_and_masks_from_partition( partition)[0] path, number_of_entries = _search_route( source_placement, destination_placement, key_and_mask, routing_tables, machine) text = "**** SubEdge '{}', from vertex: '{}' to vertex: '{}'".format( edge.label, edge.pre_subvertex.label, edge.post_subvertex.label) text += " Takes path \n {}".format(path) f_routing.write(text) f_routing.write("\n") text = "Route length: {}\n".format(number_of_entries) f_routing.write(text) # End one entry: f_routing.write("\n") progress_bar.update() f_routing.flush() f_routing.close() progress_bar.end()
def __call__(self, placements, graph, hostname, report_default_directory, write_text_specs, app_data_runtime_folder, machine, graph_mapper=None): # Keep the results dsg_targets = dict() # Keep delay extensions until the end delay_extension_placements = list() # create a progress bar for end users progress_bar = ProgressBar(len(list(placements.placements)), "Generating sPyNNaker data specifications") for placement in placements.placements: associated_vertex = graph_mapper.get_application_vertex( placement.vertex) if isinstance(associated_vertex, DelayExtensionVertex): delay_extension_placements.append( (placement, associated_vertex)) else: self._generate_data_spec_for_vertices( placement, associated_vertex, dsg_targets, hostname, report_default_directory, write_text_specs, app_data_runtime_folder, machine) progress_bar.update() for placement, associated_vertex in delay_extension_placements: self._generate_data_spec_for_vertices(placement, associated_vertex, dsg_targets, hostname, report_default_directory, write_text_specs, app_data_runtime_folder, machine) progress_bar.update() # finish the progress bar progress_bar.end() return dsg_targets
def __call__( self, placements, graph_mapper, tags, executable_finder, partitioned_graph, partitionable_graph, routing_infos, hostname, report_default_directory, write_text_specs, app_data_runtime_folder): # Keep the results executable_targets = ExecutableTargets() dsg_targets = dict() # Keep delay extensions until the end delay_extension_placements = list() # create a progress bar for end users progress_bar = ProgressBar(len(list(placements.placements)), "Generating sPyNNaker data specifications") for placement in placements.placements: associated_vertex = graph_mapper.get_vertex_from_subvertex( placement.subvertex) if isinstance(associated_vertex, DelayExtensionVertex): delay_extension_placements.append( (placement, associated_vertex)) else: self._generate_data_spec_for_subvertices( placement, associated_vertex, executable_targets, dsg_targets, graph_mapper, tags, executable_finder, partitioned_graph, partitionable_graph, routing_infos, hostname, report_default_directory, write_text_specs, app_data_runtime_folder) progress_bar.update() for placement, associated_vertex in delay_extension_placements: self._generate_data_spec_for_subvertices( placement, associated_vertex, executable_targets, dsg_targets, graph_mapper, tags, executable_finder, partitioned_graph, partitionable_graph, routing_infos, hostname, report_default_directory, write_text_specs, app_data_runtime_folder) progress_bar.update() # finish the progress bar progress_bar.end() return {'executable_targets': executable_targets, 'dsg_targets': dsg_targets}
def _run_for_core_subsets(self, core_subsets, transceiver): progress_bar = ProgressBar(len(core_subsets), "Extracting IOBUF") error_entries = list() warn_entries = list() io_buffers = list(transceiver.get_iobuf(core_subsets)) for io_buffer in io_buffers: self._check_iobuf_for_error(io_buffer, error_entries, warn_entries) progress_bar.update() progress_bar.end() return io_buffers, error_entries, warn_entries
def partitioner_report(report_folder, hostname, graph, graph_mapper): """ Generate report on the placement of sub-vertices onto cores. """ # Cycle through all vertices, and for each cycle through its sub-vertices. # For each sub-vertex, describe its core mapping. file_name = os.path.join(report_folder, "partitioned_by_vertex.rpt") f_place_by_vertex = None try: f_place_by_vertex = open(file_name, "w") except IOError: logger.error("Generate_placement_reports: Can't open file {} for" " writing.".format(file_name)) f_place_by_vertex.write( " Placement Information by Vertex\n") f_place_by_vertex.write(" ===============================\n\n") time_date_string = time.strftime("%c") f_place_by_vertex.write("Generated: {}".format(time_date_string)) f_place_by_vertex.write(" for target machine '{}'".format(hostname)) f_place_by_vertex.write("\n\n") vertices = sorted(graph.vertices, key=lambda x: x.label) progress_bar = ProgressBar(len(vertices), "Generating partitioner report") for v in vertices: vertex_name = v.label vertex_model = v.model_name num_atoms = v.n_atoms f_place_by_vertex.write( "**** Vertex: '{}'\n".format(vertex_name)) f_place_by_vertex.write("Model: {}\n".format(vertex_model)) f_place_by_vertex.write("Pop size: {}\n".format(num_atoms)) f_place_by_vertex.write("Sub-vertices: \n") partitioned_vertices = \ sorted(graph_mapper.get_subvertices_from_vertex(v), key=lambda x: x.label) partitioned_vertices = \ sorted(partitioned_vertices, key=lambda x: graph_mapper.get_subvertex_slice(x).lo_atom) for sv in partitioned_vertices: lo_atom = graph_mapper.get_subvertex_slice(sv).lo_atom hi_atom = graph_mapper.get_subvertex_slice(sv).hi_atom num_atoms = hi_atom - lo_atom + 1 my_string = " Slice {}:{} ({} atoms) \n"\ .format(lo_atom, hi_atom, num_atoms) f_place_by_vertex.write(my_string) f_place_by_vertex.flush() f_place_by_vertex.write("\n") progress_bar.update() # Close file: f_place_by_vertex.close() progress_bar.end()
def __call__(self, executable_targets, app_id, transceiver, loaded_application_data_token): """ Go through the executable targets and load each binary to \ everywhere and then send a start request to the cores that \ actually use it """ if not loaded_application_data_token: raise exceptions.ConfigurationException( "The token for having loaded the application data token is set" " to false and therefore I cannot run. Please fix and try " "again") progress_bar = ProgressBar(executable_targets.total_processors, "Loading executables onto the machine") for executable_target_key in executable_targets.binaries: file_reader = FileDataReader(executable_target_key) core_subset = executable_targets.get_cores_for_binary( executable_target_key) statinfo = os.stat(executable_target_key) size = statinfo.st_size # TODO there is a need to parse the binary and see if its # ITCM and DTCM requirements are within acceptable params for # operating on spinnaker. Currently there just a few safety # checks which may not be accurate enough. if size > constants.MAX_SAFE_BINARY_SIZE: logger.warn( "The size of {} is large enough that its" " possible that the binary may be larger than what is" " supported by spinnaker currently. Please reduce the" " binary size if it starts to behave strangely, or goes" " into the WDOG state before starting.".format( executable_target_key)) if size > constants.MAX_POSSIBLE_BINARY_SIZE: raise exceptions.ConfigurationException( "The size of {} is too large and therefore" " will very likely cause a WDOG state. Until a more" " precise measurement of ITCM and DTCM can be produced" " this is deemed as an error state. Please reduce the" " size of your binary or circumvent this error check.". format(executable_target_key)) transceiver.execute_flood(core_subset, file_reader, app_id, size) acutal_cores_loaded = 0 for chip_based in core_subset.core_subsets: for _ in chip_based.processor_ids: acutal_cores_loaded += 1 progress_bar.update(amount_to_add=acutal_cores_loaded) progress_bar.end() return {"LoadBinariesToken": True}
def get_synaptic_list_from_machine(self, graph_mapper, partitioned_graph, placements, transceiver, routing_infos): """ Get synaptic data for all connections in this Projection from the\ machine. """ if self._stored_synaptic_data_from_machine is None: timer = None if conf.config.getboolean("Reports", "display_algorithm_timings"): timer = Timer() timer.start_timing() subedges = \ graph_mapper.get_partitioned_edges_from_partitionable_edge( self) if subedges is None: subedges = list() synaptic_list = copy.copy(self._synapse_list) synaptic_list_rows = synaptic_list.get_rows() progress_bar = ProgressBar( len(subedges), "Reading back synaptic matrix for edge between" " {} and {}".format(self._pre_vertex.label, self._post_vertex.label)) for subedge in subedges: n_rows = subedge.get_n_rows(graph_mapper) pre_vertex_slice = \ graph_mapper.get_subvertex_slice(subedge.pre_subvertex) post_vertex_slice = \ graph_mapper.get_subvertex_slice(subedge.post_subvertex) sub_edge_post_vertex = \ graph_mapper.get_vertex_from_subvertex( subedge.post_subvertex) rows = sub_edge_post_vertex.get_synaptic_list_from_machine( placements, transceiver, subedge.pre_subvertex, n_rows, subedge.post_subvertex, self._synapse_row_io, partitioned_graph, routing_infos, subedge.weight_scales).get_rows() for i in range(len(rows)): synaptic_list_rows[ i + pre_vertex_slice.lo_atom].set_slice_values( rows[i], vertex_slice=post_vertex_slice) progress_bar.update() progress_bar.end() self._stored_synaptic_data_from_machine = synaptic_list if conf.config.getboolean("Reports", "display_algorithm_timings"): logger.info("Time to read matrix: {}".format( timer.take_sample())) return self._stored_synaptic_data_from_machine
def _do_allocation(self, ordered_subverts, placements, machine): # Iterate over subvertices and generate placements progress_bar = ProgressBar(len(ordered_subverts), "Placing graph vertices") resource_tracker = ResourceTracker( machine, self._generate_radial_chips(machine)) # iterate over subverts for subvertex_list in ordered_subverts: # if too many one to ones to fit on a chip, allocate individually if len(subvertex_list) > self.MAX_CORES_PER_CHIP_TO_CONSIDER: for subvertex in subvertex_list: self._allocate_individual(subvertex, placements, progress_bar, resource_tracker) else: # can allocate in one block # merge constraints placement_constraint, ip_tag_constraints, \ reverse_ip_tag_constraints = \ self._merge_constraints(subvertex_list) # locate most cores on a chip max_size_on_a_chip = resource_tracker.\ max_available_cores_on_chips_that_satisfy( placement_constraint, ip_tag_constraints, reverse_ip_tag_constraints) # if size fits block allocate, otherwise allocate individually if max_size_on_a_chip < len(subvertex_list): # collect resource requirement resources = list() for subvert in subvertex_list: resources.append(subvert.resources_required) # get cores cores = resource_tracker.allocate_group( resources, placement_constraint, ip_tag_constraints, reverse_ip_tag_constraints) # allocate cores to subverts for subvert, (x, y, p, _, _) in zip(subvertex_list, cores): placement = Placement(subvert, x, y, p) placements.add_placement(placement) progress_bar.update() else: for subvertex in subvertex_list: self._allocate_individual(subvertex, placements, progress_bar, resource_tracker) progress_bar.end()
def _run_for_placements(self, placements, transceiver): io_buffers = list() error_entries = list() warn_entries = list() progress_bar = ProgressBar(len(placements), "Extracting IOBUF") for placement in placements: iobuf = transceiver.get_iobuf_from_core(placement.x, placement.y, placement.p) io_buffers.append(iobuf) self._check_iobuf_for_error(iobuf, error_entries, warn_entries) progress_bar.update() progress_bar.end() return io_buffers, error_entries, warn_entries
def get_spikes(self, label, buffer_manager, region, state_region, placements, graph_mapper, partitionable_vertex): spike_times = list() spike_ids = list() ms_per_tick = self._machine_time_step / 1000.0 subvertices = \ graph_mapper.get_subvertices_from_vertex(partitionable_vertex) missing_str = "" progress_bar = ProgressBar(len(subvertices), "Getting spikes for {}".format(label)) for subvertex in subvertices: placement = placements.get_placement_of_subvertex(subvertex) subvertex_slice = graph_mapper.get_subvertex_slice(subvertex) x = placement.x y = placement.y p = placement.p lo_atom = subvertex_slice.lo_atom # Read the spikes n_words = int(math.ceil(subvertex_slice.n_atoms / 32.0)) n_bytes_per_block = n_words * 4 # for buffering output info is taken form the buffer manager neuron_param_region_data_pointer, data_missing = \ buffer_manager.get_data_for_vertex( placement, region, state_region) if data_missing: missing_str += "({}, {}, {}); ".format(x, y, p) raw_data = neuron_param_region_data_pointer.read_all() offset = 0 while offset < len(raw_data): ((time, n_blocks), offset) = ( struct.unpack_from("<II", raw_data, offset), offset + 8) (spike_data, offset) = (numpy.frombuffer( raw_data, dtype="uint8", count=n_bytes_per_block * n_blocks, offset=offset), offset + (n_bytes_per_block * n_blocks)) spikes = spike_data.view("<i4").byteswap().view("uint8") bits = numpy.fliplr(numpy.unpackbits(spikes).reshape( (-1, 32))).reshape((-1, n_bytes_per_block * 8)) indices = numpy.nonzero(bits)[1] times = numpy.repeat([time * ms_per_tick], len(indices)) indices = indices + lo_atom spike_ids.append(indices) spike_times.append(times) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing spike data in region {} from the" " following cores: {}".format(label, region, missing_str)) spike_ids = numpy.hstack(spike_ids) spike_times = numpy.hstack(spike_times) result = numpy.dstack((spike_ids, spike_times))[0] return result[numpy.lexsort((spike_times, spike_ids))]
def get_spikes(self, label, buffer_manager, region, state_region, placements, graph_mapper, partitionable_vertex): spike_times = list() spike_ids = list() ms_per_tick = self._machine_time_step / 1000.0 subvertices = \ graph_mapper.get_subvertices_from_vertex(partitionable_vertex) missing_str = "" progress_bar = ProgressBar(len(subvertices), "Getting spikes for {}".format(label)) for subvertex in subvertices: placement = placements.get_placement_of_subvertex(subvertex) subvertex_slice = graph_mapper.get_subvertex_slice(subvertex) x = placement.x y = placement.y p = placement.p lo_atom = subvertex_slice.lo_atom # Read the spikes n_words = int(math.ceil(subvertex_slice.n_atoms / 32.0)) n_bytes = n_words * 4 n_words_with_timestamp = n_words + 1 # for buffering output info is taken form the buffer manager neuron_param_region_data_pointer, data_missing = \ buffer_manager.get_data_for_vertex( placement, region, state_region) if data_missing: missing_str += "({}, {}, {}); ".format(x, y, p) record_raw = neuron_param_region_data_pointer.read_all() raw_data = (numpy.asarray(record_raw, dtype="uint8"). view(dtype="<i4")).reshape( [-1, n_words_with_timestamp]) split_record = numpy.array_split(raw_data, [1, 1], 1) record_time = split_record[0] * float(ms_per_tick) spikes = split_record[2].byteswap().view("uint8") bits = numpy.fliplr(numpy.unpackbits(spikes).reshape( (-1, 32))).reshape((-1, n_bytes * 8)) time_indices, indices = numpy.where(bits == 1) times = record_time[time_indices].reshape((-1)) indices = indices + lo_atom spike_ids.append(indices) spike_times.append(times) progress_bar.update() progress_bar.end() if len(missing_str) > 0: logger.warn( "Population {} is missing spike data in region {} from the" " following cores: {}".format(label, region, missing_str)) spike_ids = numpy.hstack(spike_ids) spike_times = numpy.hstack(spike_times) result = numpy.dstack((spike_ids, spike_times))[0] return result[numpy.lexsort((spike_times, spike_ids))]