def get_provenance_data_from_machine(self, transceiver, placement):
""" Get provenance from the machine
:param transceiver: spinnman interface to the machine
:param placement: the location of this vertex on the machine
"""
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
_, _, _, _, names = self._get_placement_details(placement)
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "lost_packets_without_payload"),
provenance_data[0],
report=provenance_data[0] > 0,
message=
("The live packet gatherer has lost {} packets which have "
"payloads during its execution. Try increasing the machine "
"time step or increasing the time scale factor. If you are "
"running in real time, try reducing the number of vertices "
"which are feeding this live packet gatherer".format(
provenance_data[0]))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "lost_packets_with_payload"),
provenance_data[1],
report=provenance_data[1] > 0,
message=
("The live packet gatherer has lost {} packets which do not "
"have payloads during its execution. Try increasing the "
"machine time step or increasing the time scale factor. If "
"you are running in real time, try reducing the number of "
"vertices which are feeding this live packet gatherer".format(
provenance_data[1]))))
return provenance_items
def extract_provenance(self, executor):
""" Acquires the timings from PACMAN algorithms (provenance data)
:param ~pacman.executor.PACMANAlgorithmExecutor executor:
the PACMAN workflow executor
:rtype: None
"""
for (algorithm, run_time, exec_names) in executor.algorithm_timings:
key = "run_time_of_{}".format(algorithm)
if key not in self.__already_done:
names = [self.TOP_NAME, exec_names, key]
self.__data_items.append(ProvenanceDataItem(names, run_time))
self.__already_done.add(key)
def _parse_neuron_provenance(self, label, names, provenance_data):
""" Extract and yield neuron provenance
:param str label: The label of the node
:param list(str) names: The hierarchy of names for the provenance data
:param list(int) provenance_data: A list of data items to interpret
:return: a list of provenance data items
:rtype: iterator of ProvenanceDataItem
"""
neuron_prov = NeuronProvenance(*provenance_data)
yield ProvenanceDataItem(
names + ["Last_timer_tic_the_core_ran_to"],
neuron_prov.current_timer_tick)
yield self._app_vertex.get_tdma_provenance_item(
names, label, neuron_prov.n_tdma_misses)
yield ProvenanceDataItem(
names + ["Earliest_send_time"], neuron_prov.earliest_send)
yield ProvenanceDataItem(
names + ["Latest_Send_time"], neuron_prov.latest_send)
return NeuronProvenance.N_ITEMS
def _parse_synapse_provenance(self, label, names, provenance_data):
""" Extract and yield synapse provenance
:param str label: The label of the node
:param list(str) names: The hierarchy of names for the provenance data
:param list(int) provenance_data: A list of data items to interpret
:return: a list of provenance data items
:rtype: iterator of ProvenanceDataItem
"""
synapse_prov = SynapseProvenance(*provenance_data)
yield ProvenanceDataItem(
names + [self.TOTAL_PRE_SYNAPTIC_EVENT_NAME],
synapse_prov.n_pre_synaptic_events)
yield ProvenanceDataItem(
names + [self.SATURATION_COUNT_NAME],
synapse_prov.n_saturations, synapse_prov.n_saturations > 0,
f"The weights from the synapses for {label} saturated "
f"{synapse_prov.n_saturations} times. If this causes issues you "
"can increase the spikes_per_second and / or ring_buffer_sigma "
"values located within the .spynnaker.cfg file.")
yield ProvenanceDataItem(
names + [self.SATURATED_PLASTIC_WEIGHTS_NAME],
synapse_prov.n_plastic_saturations,
synapse_prov.n_plastic_saturations > 0,
f"The weights from the plastic synapses for {label} saturated "
f"{synapse_prov.n_plastic_saturations} times. If this causes "
"issues increase the spikes_per_second and / or ring_buffer_sigma"
" values located within the .spynnaker.cfg file.")
yield ProvenanceDataItem(
names + [self.GHOST_SEARCHES], synapse_prov.n_ghost_searches,
synapse_prov.n_ghost_searches > 0,
f"The number of failed population table searches for {label} was "
f"{synapse_prov.n_ghost_searches}. If this number is large "
"relative to the predicted incoming spike rate, try increasing "
" source and target neurons per core")
yield ProvenanceDataItem(
names + [self.BIT_FIELDS_NOT_READ],
synapse_prov.n_failed_bitfield_reads, False,
f"On {label}, the filter for stopping redundant DMAs couldn't be "
f"fully filled in; it failed to read "
f"{synapse_prov.n_failed_bitfield_reads} entries. "
"Try reducing neurons per core.")
yield ProvenanceDataItem(
names + [self.INVALID_MASTER_POP_HITS],
synapse_prov.n_invalid_pop_table_hits,
synapse_prov.n_invalid_pop_table_hits > 0,
f"On {label}, there were {synapse_prov.n_invalid_pop_table_hits} "
"keys received that had no master pop entry for them. This is an "
"error, which most likely stems from bad routing.")
yield ProvenanceDataItem(
names + [self.BIT_FIELD_FILTERED_PACKETS],
synapse_prov.n_filtered_by_bitfield)
def parse_extra_provenance_items(self, label, names, provenance_data):
yield from self._parse_neuron_provenance(
label, names, provenance_data[:NeuronProvenance.N_ITEMS])
neuron_prov = NeuronMainProvenance(
*provenance_data[-NeuronMainProvenance.N_ITEMS:])
yield ProvenanceDataItem(
names + ["Timer tick overruns"], neuron_prov.n_timer_overruns,
neuron_prov.n_timer_overruns > 0,
f"Vertex {label} overran on {neuron_prov.n_timer_overruns} "
"timesteps. This may mean that the simulation results are invalid."
" Try with fewer neurons per core, increasing the time"
" scale factor, or reducing the number of spikes sent")
def _create_provenenace():
items = []
items.append(ProvenanceDataItem(["core1", "value1"], 23))
items.append(ProvenanceDataItem(["core1", "value2"], 45))
items.append(ProvenanceDataItem(["core1", "value3"], 67))
items.append(ProvenanceDataItem(["core2", "value1"], "bacon"))
items.append(ProvenanceDataItem(["core2", "value2"], 23))
items.append(ProvenanceDataItem(["core2", "value3"], 45))
return items
def _check_bit_field_router_compressor_for_success(
self, executable_targets, transceiver, host_chips,
sorter_binary_path, prov_data_items):
""" Goes through the cores checking for cores that have failed to\
generate the compressed routing tables with bitfield
:param ExecutableTargets executable_targets:
cores to load router compressor with bitfield on
:param ~.Transceiver transceiver: SpiNNMan instance
:param list(tuple(int,int)) host_chips:
the chips which need to be ran on host.
:param str sorter_binary_path: the path to the sorter binary
:param list(ProvenanceDataItem) prov_data_items:
the store of data items
:rtype: bool
"""
sorter_cores = executable_targets.get_cores_for_binary(
sorter_binary_path)
for core_subset in sorter_cores:
x = core_subset.x
y = core_subset.y
# prov names
names = list()
names.append(PROV_TOP_NAME)
names.append(PROV_CHIP_NAME.format(x, y))
names.append(MERGED_NAME)
for p in core_subset.processor_ids:
# Read the result from USER1/USER2 registers
user_1_base_address = \
transceiver.get_user_1_register_address_from_core(p)
user_2_base_address = \
transceiver.get_user_2_register_address_from_core(p)
result = struct.unpack(
"<I", transceiver.read_memory(
x, y, user_1_base_address, self._USER_BYTES))[0]
total_bit_fields_merged = struct.unpack(
"<I", transceiver.read_memory(
x, y, user_2_base_address, self._USER_BYTES))[0]
if result != self.SUCCESS:
if (x, y) not in host_chips:
host_chips.append((x, y))
return False
prov_data_items.append(ProvenanceDataItem(
names, str(total_bit_fields_merged)))
return True
def get_provenance_data(self):
name = "{}_{}_{}".format(
self._pre_population.label, self._post_population.label,
self.__class__.__name__)
return [ProvenanceDataItem(
[name, "Times_synaptic_delays_got_clipped"],
self._n_clipped_delays,
report=self._n_clipped_delays > 0,
message=(
"The delays in the connector {} from {} to {} was clipped "
"to {} a total of {} times. This can be avoided by reducing "
"the timestep or increasing the minimum delay to one "
"timestep".format(
self.__class__.__name__, self._pre_population.label,
self._post_population.label, self._min_delay,
self._n_clipped_delays)))]
def get_lut_provenance(pre_population_label, post_population_label, rule_name,
entry_name, param_name, last_entry):
# pylint: disable=too-many-arguments
top_level_name = "{}_{}_STDP_{}".format(pre_population_label,
post_population_label, rule_name)
return ProvenanceDataItem(
[top_level_name, entry_name],
last_entry,
report=last_entry > 0,
message=(
"The last entry in the STDP exponential lookup table for the {}"
" parameter of the {} between {} and {} was {} rather than 0,"
" indicating that the lookup table was not big enough at this"
" timestep and value. Try reducing the parameter value, or"
" increasing the timestep".format(param_name, rule_name,
pre_population_label,
post_population_label,
last_entry)))
def get_provenance_data(self, synapse_info):
"""
:param SynapseInformation synapse_info:
:rtype:
iterable(~spinn_front_end_common.utilities.utility_objs.ProvenanceDataItem)
"""
name = "connector_{}_{}_{}".format(synapse_info.pre_population.label,
synapse_info.post_population.label,
self.__class__.__name__)
# Convert to native Python integer; provenance system assumption
ncd = self.__n_clipped_delays.item()
yield ProvenanceDataItem([name, "Times_synaptic_delays_got_clipped"],
ncd,
report=(ncd > 0),
message=self._CLIPPED_MSG.format(
self.__class__.__name__,
synapse_info.pre_population.label,
synapse_info.post_population.label,
self.__min_delay, ncd))
def get_provenance_data_from_machine(self, transceiver, placement):
# get prov data
provenance_data = self._read_provenance_data(transceiver, placement)
# get system level prov
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
# get left over prov
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
# stuff for making prov data items
label, x, y, p, names = self._get_placement_details(placement)
# get the only app level prov item
n_buffer_overflows = provenance_data[0]
# build it
provenance_items.append(ProvenanceDataItem(
self._add_name(names, self.INPUT_BUFFER_FULL_NAME),
n_buffer_overflows, report=n_buffer_overflows > 0,
message=self.INPUT_BUFFER_FULL_MESSAGE.format(
label, x, y, p, n_buffer_overflows)))
return provenance_items
def get_provenance_data(self, synapse_info):
"""
:param SynapseInformation synapse_info:
:rtype:
list(~spinn_front_end_common.utilities.utility_objs.ProvenanceDataItem)
"""
name = "connector_{}_{}_{}".format(synapse_info.pre_population.label,
synapse_info.post_population.label,
self.__class__.__name__)
return [
ProvenanceDataItem(
[name, "Times_synaptic_delays_got_clipped"],
self.__n_clipped_delays,
report=self.__n_clipped_delays > 0,
message=
("The delays in the connector {} from {} to {} was clipped "
"to {} a total of {} times. This can be avoided by reducing "
"the timestep or increasing the minimum delay to one "
"timestep".format(self.__class__.__name__,
synapse_info.pre_population.label,
synapse_info.post_population.label,
self.__min_delay, self.__n_clipped_delays)))
]
def get_provenance_data_from_machine(self, transceiver, placement):
# get data from basic prov
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
# get item in data
queue_overflows = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.QUEUE_OVERFLOWS.value]
label, x, y, p, names = self._get_placement_details(placement)
# translate into provenance data items
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Time_queue_overflows"),
queue_overflows,
report=queue_overflows > 0,
message=(
"The packets acquired by core {}:{}:{} running model {} "
"failed to keep {} items in its buffer. Unknown how to "
"rectify".format(x, y, p, self.get_binary_file_name(),
queue_overflows))))
return provenance_items
def get_provenance_data_from_machine(self, transceiver, placement):
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
n_saturations = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.SATURATION_COUNT.value]
n_buffer_overflows = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.BUFFER_OVERFLOW_COUNT.value]
n_pre_synaptic_events = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.PRE_SYNAPTIC_EVENT_COUNT.value]
last_timer_tick = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.CURRENT_TIMER_TIC.value]
n_plastic_saturations = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.
PLASTIC_SYNAPTIC_WEIGHT_SATURATION_COUNT.value]
label, x, y, p, names = self._get_placement_details(placement)
# translate into provenance data items
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "Times_synaptic_weights_have_saturated"),
n_saturations,
report=n_saturations > 0,
message=
("The weights from the synapses for {} on {}, {}, {} saturated "
"{} times. If this causes issues you can increase the "
"spikes_per_second and / or ring_buffer_sigma "
"values located within the .spynnaker.cfg file.".format(
label, x, y, p, n_saturations))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "Times_the_input_buffer_lost_packets"),
n_buffer_overflows,
report=n_buffer_overflows > 0,
message=
("The input buffer for {} on {}, {}, {} lost packets on {} "
"occasions. This is often a sign that the system is running "
"too quickly for the number of neurons per core. Please "
"increase the timer_tic or time_scale_factor or decrease the "
"number of neurons per core.".format(label, x, y, p,
n_buffer_overflows))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "Total_pre_synaptic_events"),
n_pre_synaptic_events))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "Last_timer_tic_the_core_ran_to"),
last_timer_tick))
provenance_items.append(
ProvenanceDataItem(
self._add_name(
names, "Times_plastic_synaptic_weights_have_saturated"),
n_plastic_saturations,
report=n_plastic_saturations > 0,
message=
("The weights from the plastic synapses for {} on {}, {}, {} "
"saturated {} times. If this causes issue increase the "
"spikes_per_second and / or ring_buffer_sigma values located "
"within the .spynnaker.cfg file.".format(
label, x, y, p, n_plastic_saturations))))
return provenance_items
def __add_router_diagnostics(self, items, x, y, router_diagnostic,
reinjection_status, expected, router_table):
""" Stores router diagnostics as a set of provenance data items.
:param list(ProvenanceDataItem) items: the list to append to
:param int x: x coordinate of the router in question
:param int y: y coordinate of the router in question
:param ~.RouterDiagnostics router_diagnostic:
the router diagnostic object
:param ReInjectionStatus reinjection_status:
the data gained from the extra monitor re-injection subsystem
:param ~.MulticastRoutingTable router_table:
the router table generated by the PACMAN tools
"""
# pylint: disable=too-many-arguments
names = list()
names.append("router_provenance")
if expected:
names.append("expected_routers")
else:
names.append("unexpected_routers")
names.append("router_at_chip_{}_{}".format(x, y))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Local_Multicast_Packets"),
str(router_diagnostic.n_local_multicast_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "External_Multicast_Packets"),
str(router_diagnostic.n_external_multicast_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dropped_Multicast_Packets"),
str(router_diagnostic.n_dropped_multicast_packets),
report=(router_diagnostic.n_dropped_multicast_packets > 0
and reinjection_status is None),
message=
("The router on {}, {} has dropped {} multicast route packets. "
"Try increasing the machine_time_step and/or the time scale "
"factor or reducing the number of atoms per core.".format(
x, y, router_diagnostic.n_dropped_multicast_packets))))
items.append(
ProvenanceDataItem(
self.__add_name(
names, "Dropped_Multicast_Packets_via_local_transmission"),
str(router_diagnostic.user_3),
report=(router_diagnostic.user_3 > 0),
message=(
"The router on {}, {} has dropped {} multicast packets that"
" were transmitted by local cores. This occurs where the "
"router has no entry associated with the multi-cast key. "
"Try investigating the keys allocated to the vertices and "
"the router table entries for this chip.".format(
x, y, router_diagnostic.user_3))))
items.append(
ProvenanceDataItem(
self.__add_name(names,
"default_routed_external_multicast_packets"),
str(router_diagnostic.user_2),
report=(router_diagnostic.user_2 > 0 and
((router_table is not None
and router_table.number_of_defaultable_entries == 0)
or router_table is None)),
message=
("The router on {}, {} has default routed {} multicast packets,"
" but the router table did not expect any default routed "
"packets. This occurs where the router has no entry"
" associated with the multi-cast key. "
"Try investigating the keys allocated to the vertices and "
"the router table entries for this chip.".format(
x, y, router_diagnostic.user_2))))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Local_P2P_Packets"),
str(router_diagnostic.n_local_peer_to_peer_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "External_P2P_Packets"),
str(router_diagnostic.n_external_peer_to_peer_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dropped_P2P_Packets"),
str(router_diagnostic.n_dropped_peer_to_peer_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Local_NN_Packets"),
str(router_diagnostic.n_local_nearest_neighbour_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "External_NN_Packets"),
str(router_diagnostic.n_external_nearest_neighbour_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dropped_NN_Packets"),
str(router_diagnostic.n_dropped_nearest_neighbour_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Local_FR_Packets"),
str(router_diagnostic.n_local_fixed_route_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "External_FR_Packets"),
str(router_diagnostic.n_external_fixed_route_packets)))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dropped_FR_Packets"),
str(router_diagnostic.n_dropped_fixed_route_packets),
report=router_diagnostic.n_dropped_fixed_route_packets > 0,
message=(
"The router on chip {}:{} dropped {} Fixed route packets. "
"This is indicative of a error within the data extraction "
"process as this is the only expected user of fixed route "
"packets.".format(
x, y,
router_diagnostic.n_dropped_fixed_route_packets))))
if reinjection_status is not None:
items.append(
ProvenanceDataItem(
self.__add_name(names, "Received_For_Reinjection"),
reinjection_status.n_dropped_packets))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Missed_For_Reinjection"),
reinjection_status.n_missed_dropped_packets,
report=reinjection_status.n_missed_dropped_packets > 0,
message=(
"The extra monitor on {}, {} has missed {} "
"packets.".format(
x, y,
reinjection_status.n_missed_dropped_packets))))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Reinjection_Overflows"),
reinjection_status.n_dropped_packet_overflows,
report=reinjection_status.n_dropped_packet_overflows > 0,
message=(
"The extra monitor on {}, {} has dropped {} "
"packets.".format(
x, y,
reinjection_status.n_dropped_packet_overflows))))
items.append(
ProvenanceDataItem(self.__add_name(names, "Reinjected"),
reinjection_status.n_reinjected_packets))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dumped_from_a_Link"),
str(reinjection_status.n_link_dumps),
report=(reinjection_status.n_link_dumps > 0
and self._has_virtual_chip_connected(
self._machine, x, y)),
message=
("The extra monitor on {}, {} has detected that {} packets "
"were dumped from a outgoing link of this chip's router."
" This often occurs when external devices are used in the "
"script but not connected to the communication fabric "
"correctly. These packets may have been reinjected "
"multiple times and so this number may be a overestimate."
.format(x, y, reinjection_status.n_link_dumps))))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Dumped_from_a_processor"),
str(reinjection_status.n_processor_dumps),
report=reinjection_status.n_processor_dumps > 0,
message=
("The extra monitor on {}, {} has detected that {} packets "
"were dumped from a core failing to take the packet."
" This often occurs when the executable has crashed or"
" has not been given a multicast packet callback. It can"
" also result from the core taking too long to process"
" each packet. These packets were reinjected and so this"
" number is likely a overestimate.".format(
x, y, reinjection_status.n_processor_dumps))))
items.append(
ProvenanceDataItem(
self.__add_name(names, "Error status"),
str(router_diagnostic.error_status),
report=router_diagnostic.error_status > 0,
message=
("The router on {}, {} has a non-zero error status. This could"
" indicate a hardware fault. The errors set are {}, and the"
" error count is {}".format(x, y,
router_diagnostic.errors_set,
router_diagnostic.error_count))))
def _parse_spike_processing_fast_provenance(
self, label, names, provenance_data):
""" Extract and yield spike processing provenance
:param str label: The label of the node
:param list(str) names: The hierarchy of names for the provenance data
:param list(int) provenance_data: A list of data items to interpret
:return: a list of provenance data items
:rtype: iterator of ProvenanceDataItem
"""
prov = SpikeProcessingFastProvenance(*provenance_data)
yield ProvenanceDataItem(
names + [self.INPUT_BUFFER_FULL_NAME],
prov.n_buffer_overflows,
prov.n_buffer_overflows > 0,
f"The input buffer for {label} lost packets on "
f"{prov.n_buffer_overflows} occasions. This is often a "
"sign that the system is running too quickly for the number of "
"neurons per core. Please increase the timer_tic or"
" time_scale_factor or decrease the number of neurons per core.")
yield ProvenanceDataItem(
names + [self.DMA_COMPLETE], prov.n_dmas_complete)
yield ProvenanceDataItem(
names + [self.SPIKES_PROCESSED],
prov.n_spikes_processed)
yield ProvenanceDataItem(
names + [self.N_REWIRES_NAME], prov.n_rewires)
late_message = (
f"On {label}, {prov.n_late_packets} packets were dropped "
"from the input buffer, because they arrived too late to be "
"processed in a given time step. Try increasing the "
"time_scale_factor located within the .spynnaker.cfg file or in "
"the pynn.setup() method."
if self._app_vertex.drop_late_spikes else
f"On {label}, {prov.n_late_packets} packets arrived too "
"late to be processed in a given time step. Try increasing the "
"time_scale_factor located within the .spynnaker.cfg file or in "
"the pynn.setup() method.")
yield ProvenanceDataItem(
names + [self.N_LATE_SPIKES_NAME], prov.n_late_packets,
prov.n_late_packets > 0, late_message)
yield ProvenanceDataItem(
names + [self.MAX_FILLED_SIZE_OF_INPUT_BUFFER_NAME],
prov.max_size_input_buffer, report=False)
yield ProvenanceDataItem(
names + [self.MAX_SPIKES_RECEIVED], prov.max_spikes_received)
yield ProvenanceDataItem(
names + [self.MAX_SPIKES_PROCESSED], prov.max_spikes_processed)
yield ProvenanceDataItem(
names + [self.N_TRANSFER_TIMER_OVERRUNS],
prov.n_transfer_timer_overruns, prov.n_transfer_timer_overruns > 0,
f"On {label}, the transfer of synaptic inputs to SDRAM did not end"
" before the next timer tick started"
f" {prov.n_transfer_timer_overruns} times with a maximum overrun"
f" of {prov.max_transfer_timer_overrun}. Try increasing "
" transfer_overhead_clocks in your .spynnaker.cfg file.")
yield ProvenanceDataItem(
names + [self.N_SKIPPED_TIME_STEPS], prov.n_skipped_time_steps,
prov.n_skipped_time_steps > 0,
f"On {label}, synaptic processing did not start on"
f" {prov.n_skipped_time_steps} time steps. Try increasing the "
"time_scale_factor located within the .spynnaker.cfg file or in "
"the pynn.setup() method.")
yield ProvenanceDataItem(
names + [self.MAX_TRANSFER_TIMER_OVERRUNS],
prov.max_transfer_timer_overrun)
def _read_basic_provenance_items(self, provenance_data, placement):
transmission_event_overflow = provenance_data[
self.PROVENANCE_DATA_ENTRIES.TRANSMISSION_EVENT_OVERFLOW.value]
callback_queue_overloaded = provenance_data[
self.PROVENANCE_DATA_ENTRIES.CALLBACK_QUEUE_OVERLOADED.value]
dma_queue_overloaded = provenance_data[
self.PROVENANCE_DATA_ENTRIES.DMA_QUEUE_OVERLOADED.value]
number_of_times_timer_tic_over_ran = provenance_data[
self.PROVENANCE_DATA_ENTRIES.TIMER_TIC_HAS_OVERRUN.value]
max_number_of_times_timer_tic_over_ran = provenance_data[
self.PROVENANCE_DATA_ENTRIES.MAX_NUMBER_OF_TIMER_TIC_OVERRUN.value]
# create provenance data items for returning
label, x, y, p, names = self._get_placement_details(placement)
data_items = list()
data_items.append(
ProvenanceDataItem(
self._add_name(names,
"Times_the_transmission_of_spikes_overran"),
transmission_event_overflow,
report=transmission_event_overflow != 0,
message=(
"The transmission buffer for {} on {}, {}, {} was blocked "
"on {} occasions. This is often a sign that the system is "
"experiencing back pressure from the communication fabric. "
"Please either: "
"1. spread the load over more cores, "
"2. reduce your peak transmission load,"
"3. adjust your mapping algorithm.".format(
label, x, y, p, transmission_event_overflow))))
data_items.append(
ProvenanceDataItem(
self._add_name(names,
"Times_the_callback_queue_was_overloaded"),
callback_queue_overloaded,
report=callback_queue_overloaded != 0,
message=
("The callback queue for {} on {}, {}, {} overloaded on {} "
"occasions. This is often a sign that the system is running "
"too quickly for the number of neurons per core. Please "
"increase the machine time step or time_scale_factor or "
"decrease the number of neurons per core.".format(
label, x, y, p, callback_queue_overloaded))))
data_items.append(
ProvenanceDataItem(
self._add_name(names, "Times_the_dma_queue_was_overloaded"),
dma_queue_overloaded,
report=dma_queue_overloaded != 0,
message=
("The DMA queue for {} on {}, {}, {} overloaded on {} "
"occasions. This is often a sign that the system is running "
"too quickly for the number of neurons per core. Please "
"increase the machine time step or time_scale_factor or "
"decrease the number of neurons per core.".format(
label, x, y, p, dma_queue_overloaded))))
data_items.append(
ProvenanceDataItem(
self._add_name(names, "Times_the_timer_tic_over_ran"),
number_of_times_timer_tic_over_ran,
report=number_of_times_timer_tic_over_ran > 4,
message=
("A Timer tick callback was still executing when the next "
"timer tick callback was fired off for {} on {}, {}, {}, {} "
"times. This is a sign of the system being overloaded and "
"therefore the results are likely incorrect. Please increase "
"the machine time step or time_scale_factor or decrease the "
"number of neurons per core".format(
label, x, y, p, number_of_times_timer_tic_over_ran))))
data_items.append(
ProvenanceDataItem(
self._add_name(names,
"max_number_of_times_timer_tic_over_ran"),
max_number_of_times_timer_tic_over_ran,
report=max_number_of_times_timer_tic_over_ran > 4,
message=
("The timer for {} on {}, {}, {} fell behind by up to {} "
"ticks. This is a sign of the system being overloaded and "
"therefore the results are likely incorrect. Please increase "
"the machine time step or time_scale_factor or decrease the "
"number of neurons per core".format(
label, x, y, p, max_number_of_times_timer_tic_over_ran))))
return data_items
def get_provenance_data_from_machine(self, transceiver, placement):
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
times_timer_tic_overran = 0
for item in provenance_items:
if item.names[-1] == self._TIMER_TICK_OVERRUN:
times_timer_tic_overran = item.value
n_saturations = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.SATURATION_COUNT.value]
n_buffer_overflows = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.BUFFER_OVERFLOW_COUNT.value]
n_pre_synaptic_events = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.PRE_SYNAPTIC_EVENT_COUNT.value]
last_timer_tick = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.CURRENT_TIMER_TIC.value]
n_plastic_saturations = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.
PLASTIC_SYNAPTIC_WEIGHT_SATURATION_COUNT.value]
n_ghost_searches = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.GHOST_POP_TABLE_SEARCHES.value]
failed_to_read_bit_fields = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.FAILED_TO_READ_BIT_FIELDS.value]
dma_completes = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.DMA_COMPLETES.value]
spike_processing_count = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.SPIKE_PROGRESSING_COUNT.value]
invalid_master_pop_hits = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.INVALID_MASTER_POP_HITS.value]
n_packets_filtered_by_bit_field_filter = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.BIT_FIELD_FILTERED_COUNT.value]
n_rewires = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_REWIRES.value]
n_late_packets = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_LATE_SPIKES.value]
input_buffer_max_filled_size = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.INPUT_BUFFER_FILLED_SIZE.value]
tdma_misses = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.TDMA_MISSES.value]
label, x, y, p, names = self._get_placement_details(placement)
# translate into provenance data items
provenance_items.append(
ProvenanceDataItem(self._add_name(names,
self.SATURATION_COUNT_NAME),
n_saturations,
report=n_saturations > 0,
message=self.SATURATION_COUNT_MESSAGE.format(
label, x, y, p, n_saturations)))
provenance_items.append(
ProvenanceDataItem(self._add_name(names,
self.INPUT_BUFFER_FULL_NAME),
n_buffer_overflows,
report=n_buffer_overflows > 0,
message=self.INPUT_BUFFER_FULL_MESSAGE.format(
label, x, y, p, n_buffer_overflows)))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.TOTAL_PRE_SYNAPTIC_EVENT_NAME),
n_pre_synaptic_events))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.LAST_TIMER_TICK_NAME),
last_timer_tick))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.SATURATED_PLASTIC_WEIGHTS_NAME),
n_plastic_saturations,
report=n_plastic_saturations > 0,
message=self.SATURATED_PLASTIC_WEIGHTS_MESSAGE.format(
label, x, y, p, n_plastic_saturations)))
provenance_items.append(
ProvenanceDataItem(self._add_name(names, self.N_RE_WIRES_NAME),
n_rewires))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.GHOST_SEARCHES),
n_ghost_searches,
report=n_ghost_searches > 0,
message=
("The number of failed population table searches for {} on {},"
" {}, {} was {}. If this number is large relative to the "
"predicted incoming spike rate, try increasing source and "
"target neurons per core".format(label, x, y, p,
n_ghost_searches))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.BIT_FIELDS_NOT_READ),
failed_to_read_bit_fields,
report=False,
message=
("The filter for stopping redundant DMA's couldn't be fully "
"filled in, it failed to read {} entries, which means it "
"required a max of {} extra bytes of DTCM (assuming cores "
"have at max 255 neurons. Try reducing neurons per core, or "
"size of buffers, or neuron params per neuron etc.".format(
failed_to_read_bit_fields, failed_to_read_bit_fields *
self.WORDS_TO_COVER_256_ATOMS))))
provenance_items.append(
ProvenanceDataItem(self._add_name(names, self.DMA_COMPLETE),
dma_completes))
provenance_items.append(
ProvenanceDataItem(self._add_name(names, self.SPIKES_PROCESSED),
spike_processing_count))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, self.INVALID_MASTER_POP_HITS),
invalid_master_pop_hits,
report=invalid_master_pop_hits > 0,
message=
("There were {} keys which were received by core {}:{}:{} which"
" had no master pop entry for it. This is a error, which most "
"likely strives from bad routing.".format(
invalid_master_pop_hits, x, y, p))))
provenance_items.append((ProvenanceDataItem(
self._add_name(names, self.BIT_FIELD_FILTERED_PACKETS),
n_packets_filtered_by_bit_field_filter,
report=(n_packets_filtered_by_bit_field_filter > 0 and
(n_buffer_overflows > 0 or times_timer_tic_overran > 0)),
message=(
"There were {} packets received by {}:{}:{} that were "
"filtered by the Bitfield filterer on the core. These packets "
"were having to be stored and processed on core, which means "
"the core may not be running as efficiently as it could. "
"Please adjust the network or the mapping so that these "
"packets are filtered in the router to improve "
"performance.".format(n_packets_filtered_by_bit_field_filter,
x, y, p)))))
late_message = (self._N_LATE_SPIKES_MESSAGE_DROP
if self.__drop_late_spikes else
self._N_LATE_SPIKES_MESSAGE_NO_DROP)
provenance_items.append(
ProvenanceDataItem(self._add_name(names, self._N_LATE_SPIKES_NAME),
n_late_packets,
report=n_late_packets > 0,
message=late_message.format(
n_late_packets, label, x, y, p)))
provenance_items.append(
ProvenanceDataItem(self._add_name(
names, self._MAX_FILLED_SIZE_OF_INPUT_BUFFER_NAME),
input_buffer_max_filled_size,
report=False))
provenance_items.append(
self._app_vertex.get_tdma_provenance_item(names, x, y, p,
tdma_misses))
return provenance_items
def parse_extra_provenance_items(self, label, names, provenance_data):
(n_received, n_processed, n_added, n_sent, n_overflows, n_delays,
n_tdma_behind, n_sat, n_bad_neuron, n_bad_keys, n_late_spikes, max_bg,
n_bg_overloads) = provenance_data
# translate into provenance data items
yield ProvenanceDataItem(
names + [self.COUNT_SATURATION_NAME], n_sat, (n_sat != 0),
f"The delay extension {label} has dropped {n_sat} packets because "
"during certain time steps a neuron was asked to spike more than "
"256 times. This causes a saturation on the count tracker which "
"is a uint8. Reduce the packet rates, or modify the delay "
"extension to have larger counters.")
yield ProvenanceDataItem(
names + [self.INVALID_NEURON_ID_COUNT_NAME], n_bad_neuron,
(n_bad_neuron != 0),
f"The delay extension {label} has dropped {n_bad_neuron} packets "
"because their neuron id was not valid. This is likely a routing "
"issue. Please fix and try again")
yield ProvenanceDataItem(
names + [self.INVALID_KEY_COUNT_NAME], n_bad_keys,
(n_bad_keys != 0),
f"The delay extension {label} has dropped {n_bad_keys} packets "
"due to the packet key being invalid. This is likely a routing "
"issue. Please fix and try again")
yield ProvenanceDataItem(names + [self.N_PACKETS_RECEIVED_NAME],
n_received)
yield ProvenanceDataItem(
names + [self.N_PACKETS_PROCESSED_NAME], n_processed,
(n_received != n_processed),
f"The delay extension {label} only processed {n_processed} of "
f"{n_received} received packets. This could indicate a fault.")
yield ProvenanceDataItem(
names + [self.MISMATCH_ADDED_FROM_PROCESSED_NAME], n_added,
(n_added != n_processed),
f"The delay extension {label} only added {n_added} of "
f"{n_processed} processed packets. This could indicate a "
"routing or filtering fault")
yield ProvenanceDataItem(names + [self.N_PACKETS_SENT_NAME], n_sent)
yield ProvenanceDataItem(
names + [self.INPUT_BUFFER_LOST_NAME], n_overflows,
(n_overflows > 0),
f"The input buffer for {label} lost packets on {n_overflows} "
"occasions. This is often a sign that the system is running "
"too quickly for the number of neurons per core. Please "
"increase the timer_tic or time_scale_factor or decrease the "
"number of neurons per core.")
yield ProvenanceDataItem(names + [self.DELAYED_FOR_TRAFFIC_NAME],
n_delays)
yield self._app_vertex.get_tdma_provenance_item(
names, label, n_tdma_behind)
late_message = (
f"On {label}, {n_late_spikes} packets were dropped from the "
"input buffer, because they arrived too late to be processed in "
"a given time step. Try increasing the time_scale_factor located "
"within the .spynnaker.cfg file or in the pynn.setup() method."
if self._app_vertex.drop_late_spikes else
f"On {label}, {n_late_spikes} packets arrived too late to be "
"processed in a given time step. Try increasing the "
"time_scale_factor located within the .spynnaker.cfg file or in "
"the pynn.setup() method.")
yield ProvenanceDataItem(names + [self.N_LATE_SPIKES_NAME],
n_late_spikes,
report=(n_late_spikes > 0),
message=late_message)
yield ProvenanceDataItem(
names + [self.BACKGROUND_MAX_QUEUED_NAME], max_bg, (max_bg > 1),
f"On {label}, a maximum of {max_bg} background tasks were queued. "
"Try increasing the time_scale_factor located within the "
".spynnaker.cfg file or in the pynn.setup() method.")
yield ProvenanceDataItem(
names + [self.BACKGROUND_OVERLOADS_NAME], n_bg_overloads,
(n_bg_overloads > 0),
f"On {label}, the background queue overloaded {n_bg_overloads} "
"times. Try increasing the time_scale_factor located within the "
".spynnaker.cfg file or in the pynn.setup() method.")
def get_provenance_data_from_machine(self, transceiver, placement):
# pylint: disable=too-many-locals
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
n_packets_received = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_PACKETS_RECEIVED.value]
n_packets_processed = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_PACKETS_PROCESSED.value]
n_packets_added = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_PACKETS_ADDED.value]
n_packets_sent = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_PACKETS_SENT.value]
n_buffer_overflows = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_BUFFER_OVERFLOWS.value]
n_delays = provenance_data[
self.EXTRA_PROVENANCE_DATA_ENTRIES.N_DELAYS.value]
label, x, y, p, names = self._get_placement_details(placement)
# translate into provenance data items
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Number_of_packets_received"),
n_packets_received))
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Number_of_packets_processed"),
n_packets_processed,
report=n_packets_received != n_packets_processed,
message=(
"The delay extension {} on {}, {}, {} only processed {} of {}"
" received packets. This could indicate a fault.".format(
label, x, y, p, n_packets_processed, n_packets_received))))
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Number_of_packets_added_to_delay_slot"),
n_packets_added,
report=n_packets_added != n_packets_processed,
message=(
"The delay extension {} on {}, {}, {} only added {} of {}"
" processed packets. This could indicate a routing or"
" filtering fault".format(
label, x, y, p, n_packets_added, n_packets_processed))))
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Number_of_packets_sent"),
n_packets_sent))
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Times_the_input_buffer_lost_packets"),
n_buffer_overflows,
report=n_buffer_overflows > 0,
message=(
"The input buffer for {} on {}, {}, {} lost packets on {} "
"occasions. This is often a sign that the system is running "
"too quickly for the number of neurons per core. Please "
"increase the timer_tic or time_scale_factor or decrease the "
"number of neurons per core.".format(
label, x, y, p, n_buffer_overflows))))
provenance_items.append(ProvenanceDataItem(
self._add_name(names, "Number_of_times_delayed_to_spread_traffic"),
n_delays))
return provenance_items
def get_provenance_data_from_machine(self, transceiver, placement):
provenance_data = self._read_provenance_data(transceiver, placement)
provenance_items = self._read_basic_provenance_items(
provenance_data, placement)
provenance_data = self._get_remaining_provenance_data_items(
provenance_data)
_, _, _, _, names = self._get_placement_details(placement)
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "received_sdp_packets"),
provenance_data[
self._PROVENANCE_ITEMS.N_RECEIVED_PACKETS.value],
report=(provenance_data[
self._PROVENANCE_ITEMS.N_RECEIVED_PACKETS.value] == 0
and self._send_buffer_times is None),
message=
("No SDP packets were received by {}. If you expected packets"
" to be injected, this could indicate an error".format(
self._label))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "send_multicast_packets"),
provenance_data[self._PROVENANCE_ITEMS.N_SENT_PACKETS.value],
report=provenance_data[
self._PROVENANCE_ITEMS.N_SENT_PACKETS.value] == 0,
message=(
"No multicast packets were sent by {}. If you expected"
" packets to be sent this could indicate an error".format(
self._label))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "incorrect_keys"),
provenance_data[self._PROVENANCE_ITEMS.INCORRECT_KEYS.value],
report=provenance_data[
self._PROVENANCE_ITEMS.INCORRECT_KEYS.value] > 0,
message=(
"Keys were received by {} that did not match the key {} and"
" mask {}".format(self._label, self._virtual_key,
self._mask))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "incorrect_packets"),
provenance_data[
self._PROVENANCE_ITEMS.INCORRECT_PACKETS.value],
report=provenance_data[
self._PROVENANCE_ITEMS.INCORRECT_PACKETS.value] > 0,
message=(
"SDP Packets were received by {} that were not correct".
format(self._label))))
provenance_items.append(
ProvenanceDataItem(
self._add_name(names, "late_packets"),
provenance_data[self._PROVENANCE_ITEMS.LATE_PACKETS.value],
report=provenance_data[
self._PROVENANCE_ITEMS.LATE_PACKETS.value] > 0,
message=(
"SDP Packets were received by {} that were too late to be"
" transmitted in the simulation".format(self._label))))
return provenance_items
