def __call__(self, router_tables):
tables = MulticastRoutingTables()
previous_masks = dict()
progress = ProgressBar(
len(router_tables.routing_tables) * 2,
"Compressing Routing Tables")
# Create all masks without holes
allowed_masks = [_32_BITS - ((2**i) - 1) for i in range(33)]
# Check that none of the masks have "holes" e.g. 0xFFFF0FFF has a hole
for router_table in router_tables.routing_tables:
for entry in router_table.multicast_routing_entries:
if entry.mask not in allowed_masks:
raise PacmanRoutingException(
"Only masks without holes are allowed in tables for"
" MallocBasedRouteMerger (disallowed mask={})".format(
hex(entry.mask)))
for router_table in progress.over(router_tables.routing_tables):
new_table = self._merge_routes(router_table, previous_masks)
tables.add_routing_table(new_table)
n_entries = len([
entry for entry in new_table.multicast_routing_entries
if not entry.defaultable
])
print("Reduced from {} to {}".format(
len(router_table.multicast_routing_entries), n_entries))
if n_entries > 1023:
raise PacmanRoutingException(
"Cannot make table small enough: {} entries".format(
n_entries))
return tables
def _update_neighbour(tables, neighbour, current, source, weight):
""" Update the lowest cost for each neighbour_xy of a node
:param dict(tuple(int,int),_DijkstraInfo) tables:
:param ~spinn_machine.Link neighbour:
:param tuple(int,int) current:
:param tuple(int,int) source:
:param float weight:
:raise PacmanRoutingException: when the algorithm goes to a node that
doesn't exist in the machine or the node's cost was set too low.
"""
neighbour_xy = (neighbour.destination_x, neighbour.destination_y)
if neighbour_xy not in tables:
raise PacmanRoutingException(
"Tried to propagate to ({}, {}), which is not in the"
" graph: remove non-existent neighbours".format(
neighbour.destination_x, neighbour.destination_y))
chip_cost = tables[current].cost
neighbour_cost = tables[neighbour_xy].cost
# Only try to update if the neighbour_xy is within the graph and the
# cost if the node hasn't already been activated and the lowest cost
# if the new cost is less, or if there is no current cost.
new_weight = float(chip_cost + weight)
if (not tables[neighbour_xy].activated
and (neighbour_cost is None or new_weight < neighbour_cost)):
# update Dijkstra table
tables[neighbour_xy].cost = new_weight
if tables[neighbour_xy].cost == 0 and neighbour_xy != source:
raise PacmanRoutingException(
"!!!Cost of non-source node ({}, {}) was set to zero!!!".
format(neighbour.destination_x, neighbour.destination_y))
def compare_route(f, o_route, compressed_dict, o_code=None, start=0):
if o_code is None:
o_code = codify(o_route)
keys = list(compressed_dict.keys())
for i in range(start, len(keys)):
c_code = keys[i]
if covers(o_code, c_code):
c_route = compressed_dict[c_code]
f.write("\t\t{}\n".format(reports.format_route(c_route)))
if o_route.defaultable != c_route.defaultable:
raise PacmanRoutingException(
"Compressed route {} covers original route {} but has "
"a different defaultable value.".format(c_route, o_route))
if o_route.processor_ids != c_route.processor_ids:
raise PacmanRoutingException(
"Compressed route {} covers original route {} but has "
"a different processor_ids.".format(c_route, o_route))
if o_route.link_ids != c_route.link_ids:
raise PacmanRoutingException(
"Compressed route {} covers original route {} but has "
"a different link_ids.".format(c_route, o_route))
remainders = calc_remainders(o_code, c_code)
for remainder in remainders:
compare_route(f,
o_route,
compressed_dict,
o_code=remainder,
start=i + 1)
return
compare_route(f, o_route, compressed_dict, o_code=o_code, start=i + 1)
return
def compare_route(o_route, compressed_dict, o_code=None, start=0):
if o_code is None:
o_code = codify(o_route)
keys = list(compressed_dict.keys())
for i in range(start, len(keys)):
c_code = keys[i]
print(o_code, c_code) # TODO: Don't print this message!
if covers(o_code, c_code):
print("covers") # TODO: Don't print this message!
c_route = compressed_dict[c_code]
if o_route.defaultable != c_route.defaultable:
PacmanRoutingException( # TODO: Raise this exception!
"Compressed route {} covers original route {} but has "
"a different defaultable value.".format(c_route, o_route))
if o_route.processor_ids != c_route.processor_ids:
PacmanRoutingException( # TODO: Raise this exception!
"Compressed route {} covers original route {} but has "
"a different processor_ids.".format(c_route, o_route))
if o_route.link_ids != c_route.link_ids:
PacmanRoutingException( # TODO: Raise this exception!
"Compressed route {} covers original route {} but has "
"a different link_ids.".format(c_route, o_route))
remainders = calc_remainders(o_code, c_code)
print(remainders) # TODO: Don't print this message!
for remainder in remainders:
compare_route(o_route,
compressed_dict,
o_code=remainder,
start=i + 1)
return
compare_route(o_route, compressed_dict, o_code=o_code, start=i + 1)
return
def compare_route(o_route, compressed_dict, o_code=None, start=0, f=None):
"""
:param ~spinn_machine.MulticastRoutingEntry o_route: the original route
:param dict compressed_dict:
:param str o_code:
:param int start:
:param ~io.FileIO f:
:rtype: None
"""
if o_code is None:
o_code = codify(o_route)
keys = list(compressed_dict.keys())
for i in range(start, len(keys)):
c_code = keys[i]
if covers(o_code, c_code):
c_route = compressed_dict[c_code]
if f is not None:
f.write("\t\t{}\n".format(format_route(c_route)))
if o_route.processor_ids != c_route.processor_ids:
if set(o_route.processor_ids) != set(c_route.processor_ids):
raise PacmanRoutingException(
"Compressed route {} covers original route {} but has "
"a different processor_ids.".format(c_route, o_route))
if o_route.link_ids != c_route.link_ids:
if set(o_route.link_ids) != set(c_route.link_ids):
raise PacmanRoutingException(
"Compressed route {} covers original route {} but has "
"a different link_ids.".format(c_route, o_route))
if not o_route.defaultable and c_route.defaultable:
if o_route == c_route:
raise PacmanRoutingException(
"Compressed route {} while original route {} but has "
"a different defaultable value.".format(
c_route, o_route))
else:
compare_route(o_route,
compressed_dict,
o_code=o_code,
start=i + 1,
f=f)
else:
remainders = calc_remainders(o_code, c_code)
for remainder in remainders:
compare_route(o_route,
compressed_dict,
o_code=remainder,
start=i + 1,
f=f)
return
if not o_route.defaultable:
# print("No route found {}".format(o_route))
raise PacmanRoutingException("No route found {}".format(o_route))
def get_multicast_routing_entry_by_routing_entry_key(
self, routing_entry_key, mask):
""" Get the routing entry associated with the specified key_combo-mask\
combination or None if the routing table does not match the\
key_combo
:param routing_entry_key: the routing key to be searched
:type routing_entry_key: int
:param mask: the routing mask to be searched
:type mask: int
:return: the routing entry associated with the routing key_combo or\
None if no such entry exists
:rtype:\
:py:class:`spinn_machine.MulticastRoutingEntry`
"""
if (routing_entry_key & mask) != routing_entry_key:
raise PacmanRoutingException(
"The key {} is changed when masked with the mask {}."
" This is determined to be an error in the tool chain. Please "
"correct this and try again.".format(routing_entry_key, mask))
tuple_key = (routing_entry_key, mask)
if tuple_key in self._multicast_routing_entries_by_routing_entry_key:
return self._multicast_routing_entries_by_routing_entry_key[
tuple_key]
return None
def _locate_routing_entry(current_router, key, n_atoms):
""" Locate the entry from the router based off the edge
:param MulticastRoutingTable current_router:
the current router being used in the trace
:param int key: the key being used by the source placement
:param int n_atoms: the number of atoms
:rtype: ~spinn_machine.MulticastRoutingEntry
:raise PacmanRoutingException:
when there is no entry located on this router
"""
found_entry = None
for entry in current_router.multicast_routing_entries:
key_combo = entry.mask & key
e_key = entry.routing_entry_key
if key_combo == e_key:
if found_entry is None:
found_entry = entry
else:
logger.warning(
"Found more than one entry for key {}. This could be "
"an error, as currently no router supports overloading"
" of entries.", hex(key))
if entry.mask in range_masks:
last_atom = key + n_atoms - 1
last_key = e_key + (~entry.mask & FULL_MASK)
if last_key < last_atom:
raise PacmanRoutingException(
"Full key range not covered: key:{} key_combo:{} "
"mask:{}, last_key:{}, e_key:{}".format(
hex(key), hex(key_combo), hex(entry.mask),
hex(last_key), hex(e_key)))
elif entry.mask in range_masks:
last_atom = key + n_atoms
last_key = e_key + (~entry.mask & FULL_MASK)
if min(last_key, last_atom) - max(e_key, key) + 1 > 0:
raise Exception(
"Key range partially covered: key:{} key_combo:{} "
"mask:{}, last_key:{}, e_key:{}".format(
hex(key), hex(key_combo), hex(entry.mask),
hex(last_key), hex(e_key)))
if found_entry is None:
raise PacmanRoutingException("no entry located")
return found_entry
def _check_visited_routers(chip_x, chip_y, visited_routers):
""" Check if the trace has visited this router already
:param int chip_x: the x coordinate of the chip being checked
:param int chip_y: the y coordinate of the chip being checked
:param set(tuple(int,int)) visited_routers: routers already visited
:rtype: None
:raise PacmanRoutingException: when a router has been visited twice.
"""
visited_routers_router = (chip_x, chip_y)
if visited_routers_router in visited_routers:
raise PacmanRoutingException(
"visited this router before, there is a cycle here. "
"The routers I've currently visited are {} and the router i'm "
"visiting is {}".format(visited_routers, visited_routers_router))
visited_routers.add(visited_routers_router)
def _do_fixed_routing(self, ethernet_connected_chip):
""" Handles this board through the quick routing process, based on a\
predefined routing table.
:param ~spinn_machine.Chip ethernet_connected_chip:
the Ethernet connected chip
:raises PacmanRoutingException:
:raises PacmanAlreadyExistsException:
"""
eth_x = ethernet_connected_chip.x
eth_y = ethernet_connected_chip.y
to_route = set(self._machine.get_existing_xys_by_ethernet(
eth_x, eth_y))
routed = set()
routed.add((eth_x, eth_y))
to_route.remove((eth_x, eth_y))
while len(to_route) > 0:
found = []
for x, y in to_route:
# Check links starting with the most direct to 0,0
for link_id in [4, 3, 5, 2, 0, 1]:
# Get protential destination
destination = self._machine.xy_over_link(x, y, link_id)
# If it is useful
if destination in routed:
# check it actually exits
if self._machine.is_link_at(x, y, link_id):
# build entry and add to table and add to tables
key = (x, y)
self.__add_fixed_route_entry(key, [link_id], [])
found.append(key)
break
if len(found) == 0:
raise PacmanRoutingException(
"Unable to do fixed point routing on {}.".format(
ethernet_connected_chip.ip_address))
for key in found:
to_route.remove(key)
routed.add(key)
# create final fixed route entry
# locate where to put data on ethernet chip
processor_id = self.__locate_destination(ethernet_connected_chip)
# build entry and add to table and add to tables
self.__add_fixed_route_entry((eth_x, eth_y), [], [processor_id])
def _minimum(tables):
# This is the lowest cost across ALL deactivated nodes in the graph.
lowest_cost = sys.maxsize
lowest = None
# Find the next node to be activated
for key in tables:
# Don't continue if the node hasn't even been touched yet
if (tables[key].cost is not None and not tables[key].activated
and tables[key].cost < lowest_cost):
lowest_cost = tables[key].cost
lowest = key
# If there were no deactivated nodes with costs, but the destination
# was not reached this iteration, raise an exception
if lowest is None:
raise PacmanRoutingException(
"Destination could not be activated, ending run")
return int(lowest[0]), int(lowest[1])
def _check_visited_routers(chip_x, chip_y, visited_routers):
""" Check if the trace has visited this router already
:param chip_x: the x coordinate of the chip being checked
:param chip_y: the y coordinate of the chip being checked
:param visited_routers: routers already visited
:type chip_x: int
:type chip_y: int
:type visited_routers: iterable of\
:py:class:`pacman.model.routing_tables.MulticastRoutingTable`
:rtype: None
:raise PacmanRoutingException: when a router has been visited twice.
"""
visited_routers_router = (chip_x, chip_y)
if visited_routers_router in visited_routers:
raise PacmanRoutingException(
"visited this router before, there is a cycle here. "
"The routers I've currently visited are {} and the router i'm "
"visiting is {}"
.format(visited_routers, visited_routers_router))
visited_routers.add(visited_routers_router)
def _generate_routing_tree(self, ethernet_chip):
""" Generates a map for each chip to over which link it gets its data.
:param ~spinn_machine.Chip ethernet_chip:
:return: Map of chip.x, chip.y tp (source.x, source.y, source.link)
:rtype: dict(tuple(int, int), tuple(int, int, int))
"""
eth_x = ethernet_chip.x
eth_y = ethernet_chip.y
tree = dict()
to_reach = set(
self._machine.get_existing_xys_by_ethernet(eth_x, eth_y))
reached = set()
reached.add((eth_x, eth_y))
to_reach.remove((eth_x, eth_y))
found = set()
found.add((eth_x, eth_y))
while len(to_reach) > 0:
just_reached = found
found = set()
for x, y in just_reached:
# Check links starting with the most direct from 0,0
for link_id in [1, 0, 2, 5, 3, 4]:
# Get protential destination
destination = self._machine.xy_over_link(x, y, link_id)
# If it is useful
if destination in to_reach:
# check it actually exits
if self._machine.is_link_at(x, y, link_id):
# Add to tree and record chip reachable
tree[destination] = (x, y, link_id)
to_reach.remove(destination)
found.add(destination)
if len(found) == 0:
raise PacmanRoutingException(
"Unable to do data in routing on {}.".format(
ethernet_chip.ip_address))
return tree
def _search_route(source_placement, dest_placements, key_and_mask,
routing_tables, machine, n_atoms, is_continuous):
""" Locate if the routing tables work for the source to desks as\
defined
:param source_placement: the placement from which the search started
:param dest_placements: \
the placements to which this trace should visit only once
:param key_and_mask: the key and mask associated with this set of edges
:param n_atoms: the number of atoms going through this path
:param is_continuous: \
whether the keys and atoms mapping is continuous
:type source_placement: \
:py:class:`pacman.model.placements.Placement`
:type dest_placements: iterable(PlacementTuple)
:type key_and_mask: \
:py:class:`pacman.model.routing_info.BaseKeyAndMask`
:rtype: None
:raise PacmanRoutingException: when the trace completes and there are\
still destinations not visited
"""
if logger.isEnabledFor(logging.DEBUG):
for dest in dest_placements:
logger.debug("[{}:{}:{}]", dest.x, dest.y, dest.p)
located_destinations = set()
failed_to_cover_all_keys_routers = list()
_start_trace_via_routing_tables(
source_placement, key_and_mask, located_destinations,
routing_tables, machine, n_atoms, is_continuous,
failed_to_cover_all_keys_routers)
# start removing from located_destinations and check if destinations not
# reached
failed_to_reach_destinations = list()
for dest in dest_placements:
if dest in located_destinations:
located_destinations.remove(dest)
else:
failed_to_reach_destinations.append(dest)
# check for error if trace didn't reach a destination it was meant to
error_message = ""
if failed_to_reach_destinations:
output_string = ""
for dest in failed_to_reach_destinations:
output_string += "[{}:{}:{}]".format(dest.x, dest.y, dest.p)
source_processor = "[{}:{}:{}]".format(
source_placement.x, source_placement.y, source_placement.p)
error_message += ("failed to locate all destinations with vertex"
" {} on processor {} with keys {} as it did not "
"reach destinations {}".format(
source_placement.vertex.label, source_processor,
key_and_mask, output_string))
# check for error if the trace went to a destination it shouldn't have
if located_destinations:
output_string = ""
for dest in located_destinations:
output_string += "[{}:{}:{}]".format(dest.x, dest.y, dest.p)
source_processor = "[{}:{}:{}]".format(
source_placement.x, source_placement.y, source_placement.p)
error_message += ("trace went to more failed to locate all "
"destinations with vertex {} on processor {} "
"with keys {} as it didn't reach destinations {}"
.format(
source_placement.vertex.label, source_processor,
key_and_mask, output_string))
if failed_to_cover_all_keys_routers:
output_string = ""
for data_entry in failed_to_cover_all_keys_routers:
output_string += "[{}, {}, {}, {}]".format(
data_entry['router_x'], data_entry['router_y'],
data_entry['keys'], data_entry['source_mask'])
source_processor = "[{}:{}:{}]".format(
source_placement.x, source_placement.y, source_placement.p)
error_message += (
"trace detected that there were atoms which the routing entry's"
" wont cover and therefore packets will fly off to unknown places."
" These keys came from the vertex {} on processor {} and the"
" failed routers are {}".format(
source_placement.vertex.label, source_processor,
output_string))
# raise error if required
if error_message != "":
raise PacmanRoutingException(error_message)
logger.debug("successful test between {} and {}",
source_placement.vertex.label, dest_placements)
def _retrace_back_to_source(self, dest, tables, edge, nodes_info,
source_processor, graph):
"""
:param Placement dest: Destination placement
:param dict(tuple(int,int),_DijkstraInfo) tables:
:param MachineEdge edge:
:param dict(tuple(int,int),_NodeInfo) nodes_info:
:param int source_processor:
:param MachineGraph graph:
:return: the next coordinates to look into
:rtype: tuple(int, int)
:raise PacmanRoutingException:
when the algorithm doesn't find a next point to search from. AKA,
the neighbours of a chip do not have a cheaper cost than the node
itself, but the node is not the destination or when the algorithm
goes to a node that's not considered in the weighted search.
"""
# Set the tracking node to the destination to begin with
x, y = dest.x, dest.y
routing_entry_route_processors = []
# if the processor is None, don't add to router path entry
if dest.p is not None:
routing_entry_route_processors.append(dest.p)
routing_entry_route_links = None
# build the multicast entry
partitions = graph.get_multicast_edge_partitions_starting_at_vertex(
edge.pre_vertex)
prev_entry = None
for partition in partitions:
if edge in partition:
entry = MulticastRoutingTableByPartitionEntry(
out_going_links=routing_entry_route_links,
outgoing_processors=routing_entry_route_processors)
self._routing_paths.add_path_entry(entry, dest.x, dest.y,
partition)
prev_entry = entry
while tables[x, y].cost != 0:
for idx, neighbour in enumerate(nodes_info[x, y].neighbours):
if neighbour is not None:
n_xy = (neighbour.destination_x, neighbour.destination_y)
# Only check if it can be a preceding node if it actually
# exists
if n_xy not in tables:
raise PacmanRoutingException(
"Tried to trace back to node not in "
"graph: remove non-existent neighbours")
if tables[n_xy].cost is not None:
x, y, prev_entry, added = self._create_routing_entry(
n_xy, tables, idx, nodes_info, x, y, prev_entry,
edge, graph)
if added:
break
else:
raise PacmanRoutingException(
"Iterated through all neighbours of tracking node but"
" did not find a preceding node! Consider increasing "
"acceptable discrepancy between sought traceback cost"
" and actual cost at node. Terminating...")
prev_entry.incoming_processor = source_processor
return x, y