def _check_masks_are_correct(partition): """ Check that the masks between a fixed mask constraint and a fixed_field\ constraint. Raises error if not. :param AbstractSingleSourcePartition partition: the outgoing_edge_partition to search for these constraints :raise PacmanInvalidParameterException: if the masks are incompatible """ fixed_mask = locate_constraints_of_type( partition.constraints, FixedMaskConstraint)[0] fixed_field = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint)[0] mask = fixed_mask.mask for field in fixed_field.fields: if field.mask & mask != field.mask: raise PacmanInvalidParameterException( "field.mask, mask", "The field mask {} is outside of the mask {}".format( field.mask, mask), "{}:{}".format(field.mask, mask)) for other_field in fixed_field.fields: if other_field != field and other_field.mask & field.mask != 0: raise PacmanInvalidParameterException( "field.mask, mask", "Field masks {} and {} overlap".format( field.mask, other_field.mask), "{}:{}".format(field.mask, mask))
def _check_masks_are_correct(partition): """ Check that the masks between a fixed mask constraint\ and a fixed_field constraint. completes if its correct, raises error\ otherwise :param partition: \ the outgoing_edge_partition to search for these constraints :rtype: None: """ fixed_mask = locate_constraints_of_type(partition.constraints, FixedMaskConstraint)[0] fixed_field = locate_constraints_of_type(partition.constraints, FixedKeyFieldConstraint)[0] mask = fixed_mask.mask for field in fixed_field.fields: if field.mask & mask != field.mask: raise PacmanInvalidParameterException( "field.mask, mask", "The field mask {} is outside of the mask {}".format( field.mask, mask), "{}:{}".format(field.mask, mask)) for other_field in fixed_field.fields: if other_field != field and other_field.mask & field.mask != 0: raise PacmanInvalidParameterException( "field.mask, mask", "Field masks {} and {} overlap".format( field.mask, other_field.mask), "{}:{}".format(field.mask, mask))
def _check_masks_are_correct(partition): """ Check that the masks between a fixed mask constraint\ and a fixed_field constraint. completes if its correct, raises error\ otherwise :param partition: \ the outgoing_edge_partition to search for these constraints :rtype: None: """ fixed_mask = locate_constraints_of_type( partition.constraints, FixedMaskConstraint)[0] fixed_field = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint)[0] mask = fixed_mask.mask for field in fixed_field.fields: if field.mask & mask != field.mask: raise PacmanInvalidParameterException( "field.mask, mask", "The field mask {} is outside of the mask {}".format( field.mask, mask), "{}:{}".format(field.mask, mask)) for other_field in fixed_field.fields: if other_field != field and other_field.mask & field.mask != 0: raise PacmanInvalidParameterException( "field.mask, mask", "Field masks {} and {} overlap".format( field.mask, other_field.mask), "{}:{}".format(field.mask, mask))
def determine_max_atoms_for_vertex(vertex): """ returns the max atom constraint after assessing them all. :param ApplicationVertex vertex: the vertex to find max atoms of :return: the max number of atoms per core :rtype: int """ possible_max_atoms = list() n_atoms = None max_atom_constraints = utils.locate_constraints_of_type( vertex.constraints, MaxVertexAtomsConstraint) for constraint in max_atom_constraints: possible_max_atoms.append(constraint.size) n_atom_constraints = utils.locate_constraints_of_type( vertex.constraints, FixedVertexAtomsConstraint) for constraint in n_atom_constraints: if n_atoms is not None and constraint.size != n_atoms: raise PacmanPartitionException( CONTRADICTORY_FIXED_ATOM_ERROR.format(n_atoms, constraint.size)) n_atoms = constraint.size if len(possible_max_atoms) != 0: return int(min(possible_max_atoms)) else: return vertex.n_atoms
def _merge_constraints(subvertex_list): merged_placement = None ip_tag = list() reverse_ip_tags = list() for subvertex in subvertex_list: place_constraints = utility_calls.locate_constraints_of_type( subvertex.constraints, PlacerChipAndCoreConstraint) ip_tag_constraints = utility_calls.locate_constraints_of_type( subvertex.constraints, TagAllocatorRequireIptagConstraint) ip_tag.extend(ip_tag_constraints) reverse_ip_tag = utility_calls.locate_constraints_of_type( subvertex.constraints, TagAllocatorRequireReverseIptagConstraint) reverse_ip_tags.extend(reverse_ip_tag) if len(place_constraints) != 0: for place_constraint in place_constraints: if merged_placement is None: merged_placement = place_constraint else: x_level = merged_placement.x == \ place_constraint.x y_level = merged_placement.y == \ place_constraint.y p_level = merged_placement.p != \ place_constraint.p if not x_level or not y_level or not p_level: raise exceptions.PacmanConfigurationException( "can't handle conflicting constraints") return merged_placement, ip_tag, reverse_ip_tags
def __set_max_atoms_to_splitters(app_graph): """ get the constraints sorted out. :param ApplicationGraph app_graph: the app graph """ for vertex in app_graph.vertices: for constraint in utils.locate_constraints_of_type( vertex.constraints, MaxVertexAtomsConstraint): vertex.splitter.set_max_atoms_per_core(constraint.size, False) for constraint in utils.locate_constraints_of_type( vertex.constraints, FixedVertexAtomsConstraint): vertex.splitter.set_max_atoms_per_core(constraint.size, True)
def check_incoming_verts(one_to_one_verts, vertex, partitioned_graph, found_list): """ Adds subverts which have a one to one connection :param one_to_one_verts: the list of sorted vertices :param vertex: the destination vertex :param partitioned_graph: the partitioned graph :param found_list: the list of found vertices so far :return: """ # locate incoming edges for this vertex incoming_edges = \ partitioned_graph.incoming_subedges_from_subvertex(vertex) # locate constraints of chip and core for this vertex chip_constraint = utility_calls.locate_constraints_of_type( vertex.constraints, PlacerChipAndCoreConstraint) for incoming_edge in incoming_edges: incoming_vert = incoming_edge.pre_subvertex number_of_outgoing_edges = partitioned_graph.\ outgoing_subedges_from_subvertex(incoming_vert) # if only one outgoing edge, decide to put it in same chip pile if (len(number_of_outgoing_edges) == 1 and incoming_vert not in found_list): # if the vertex has no constraint, put in if len(chip_constraint) != 0: one_to_one_verts.append(incoming_vert) found_list.append(incoming_vert) else: # if constraint exists, try to satisfy constraints. chip_constraint_incoming = \ utility_calls.locate_constraints_of_type( incoming_vert.constraints, PlacerChipAndCoreConstraint) if len(chip_constraint_incoming) == 0: one_to_one_verts.append(incoming_vert) found_list.append(incoming_vert) else: x_level = chip_constraint[0].x == \ chip_constraint_incoming[0].x y_level = chip_constraint[0].y == \ chip_constraint_incoming[0].y p_level = chip_constraint[0].p != \ chip_constraint_incoming[0].p if x_level and y_level and p_level: one_to_one_verts.append(incoming_vert) found_list.append(incoming_vert)
def get_fixed_mask(same_key_group): """ Get a fixed mask from a group of edges if a\ :py:class:`FixedMaskConstraint`\ constraint exists in any of the edges in the group. :param iterable(MachineEdge) same_key_group: Set of edges that are to be assigned the same keys and masks :return: The fixed mask if found, or None :rtype: tuple(int or None, iterable(Field) or None) :raise PacmanValueError: If two edges conflict in their requirements """ mask = None fields = None edge_with_mask = None for edge in same_key_group: for constraint in locate_constraints_of_type( edge.constraints, FixedMaskConstraint): if mask is not None and mask != constraint.mask: raise PacmanValueError( "Two Edges {} and {} must have the same key and mask, " "but have different fixed masks, {} and {}".format( edge, edge_with_mask, mask, constraint.mask)) if (fields is not None and constraint.fields is not None and fields != constraint.fields): raise PacmanValueError( "Two Edges {} and {} must have the same key and mask, " "but have different field ranges".format( edge, edge_with_mask)) mask = constraint.mask edge_with_mask = edge if constraint.fields is not None: fields = constraint.fields return mask, fields
def _locate_connected_chip_data(vertex, machine): """ Finds the connected virtual chip :param vertex: :param machine: :return: """ # locate the chip from the placement constraint placement_constraint = utility_calls.locate_constraints_of_type( vertex.constraints, PlacerChipAndCoreConstraint) router = machine.get_chip_at(placement_constraint.x, placement_constraint.y).router found_link = False link_id = 0 while not found_link or link_id < 5: if router.is_link(link_id): found_link = True else: link_id += 1 if not found_link: raise exceptions.PacmanConfigurationException( "Can't find the real chip this virtual chip is connected to." "Please fix and try again.") else: return ("[{}, {}]".format( router.get_link(link_id).destination_x, router.get_link(link_id).destination_y), router.get_link(link_id).multicast_default_from)
def _locate_connected_chip_data(vertex, machine): """ Finds the connected virtual chip :param vertex: :param machine: :return: """ # locate the chip from the placement constraint placement_constraint = utility_calls.locate_constraints_of_type( vertex.constraints, PlacerChipAndCoreConstraint) router = machine.get_chip_at( placement_constraint.x, placement_constraint.y).router found_link = False link_id = 0 while not found_link or link_id < 5: if router.is_link(link_id): found_link = True else: link_id += 1 if not found_link: raise exceptions.PacmanConfigurationException( "Can't find the real chip this virtual chip is connected to." "Please fix and try again.") else: return ("[{}, {}]".format(router.get_link(link_id).destination_x, router.get_link(link_id).destination_y), router.get_link(link_id).multicast_default_from)
def _locate_vertices_to_partition_now(vertex): """ Locate any other vertices that need to be partitioned with the\ exact same ranges of atoms :param vertex: the vertex that is currently being partitioned :type vertex:\ :py:class:`pacman.model.partitionable_graph.abstract_partitionable_vertex.AbstractPartitionableVertex` :return: iterable of vertexes that need to be partitioned with the\ exact same range of atoms :rtype: iterable of\ :py:class:`pacman.model.partitionable_graph.abstract_partitionable_vertex.AbstractPartitionableVertex` :raise PacmanPartitionException: if the vertices that need to be \ partitioned the same have different numbers of atoms """ partition_together_vertices = list() partition_together_vertices.append(vertex) same_size_vertex_constraints = \ utility_calls.locate_constraints_of_type( vertex.constraints, PartitionerSameSizeAsVertexConstraint) for constraint in same_size_vertex_constraints: if constraint.vertex.n_atoms != vertex.n_atoms: raise exceptions.PacmanPartitionException( "A vertex and its partition-dependent vertices must " "have the same number of atoms") else: partition_together_vertices.append(constraint.vertex) return partition_together_vertices
def _place_vertex(self, vertex, resource_tracker, machine, placements): # Check for the radial placement constraint radial_constraints = utility_calls.locate_constraints_of_type( [vertex], PlacerRadialPlacementFromChipConstraint) start_x = 0 start_y = 0 for constraint in radial_constraints: if start_x is None: start_x = constraint.x elif start_x != constraint.x: raise PacmanPlaceException("Non-matching constraints") if start_y is None: start_y = constraint.y elif start_y != constraint.y: raise PacmanPlaceException("Non-matching constraints") chips = None if start_x is not None and start_y is not None: chips = self._generate_radial_chips(machine, resource_tracker, start_x, start_y) # Create and store a new placement (x, y, p, _, _) = resource_tracker.allocate_constrained_resources( vertex.resources_required, vertex.constraints, chips) placement = Placement(vertex, x, y, p) placements.add_placement(placement)
def _place_vertex(self, vertex, resource_tracker, machine, placements): # Check for the radial placement constraint radial_constraints = utility_calls.locate_constraints_of_type( [vertex], PlacerRadialPlacementFromChipConstraint) start_x = None start_y = None for constraint in radial_constraints: if start_x is None: start_x = constraint.x elif start_x != constraint.x: raise PacmanPlaceException("Non-matching constraints") if start_y is None: start_y = constraint.y elif start_y != constraint.y: raise PacmanPlaceException("Non-matching constraints") chips = None if start_x is not None and start_y is not None: chips = self._generate_radial_chips(machine, resource_tracker, start_x, start_y) # Create and store a new placement (x, y, p, _, _) = resource_tracker.allocate_constrained_resources( vertex.resources_required, vertex.constraints, chips) placement = Placement(vertex, x, y, p) placements.add_placement(placement)
def get_fixed_key_and_mask(same_key_group): """ Gets a fixed key and mask from a group of partitioned edges if a\ :py:class:`pacman.model.constraints.key_allocator_same_key_constraint.KeyAllocatorFixedKeyAndMaskConstraint`\ constraint exists in any of the edges in the group. :param same_key_group: Set of partitioned edges that are to be\ assigned the same keys and masks :type same_key_group: iterable of\ :py:class:`pacman.model.partitioned_graph.partitioned_edge.PartitionedEdge` :raise PacmanValueError: If two edges conflict in their requirements """ keys_and_masks = None edge_with_key_and_mask = None for edge in same_key_group: constraints = utility_calls.locate_constraints_of_type( edge.constraints, KeyAllocatorFixedKeyAndMaskConstraint) for constraint in constraints: # Check for conflicts if (keys_and_masks is not None and keys_and_masks != constraint.keys_and_masks): raise PacmanValueError( "Two Partitioned Edges {} and {} must have the same" " key and mask, but have different fixed key and" " masks, {} and {}".format( edge, edge_with_key_and_mask, keys_and_masks, constraint.keys_and_masks)) keys_and_masks = constraint.keys_and_masks edge_with_key_and_mask = edge return keys_and_masks
def _place_vertex(self, vertex, resource_tracker, machine, placements, vertices_on_same_chip): vertices = vertices_on_same_chip[vertex] # Check for the radial placement constraint radial_constraints = locate_constraints_of_type( vertices, RadialPlacementFromChipConstraint) start_x, start_y = self._get_start(radial_constraints) chips = None if start_x is not None and start_y is not None: chips = self._generate_radial_chips(machine, resource_tracker, start_x, start_y) if len(vertices) > 1: assigned_values = \ resource_tracker.allocate_constrained_group_resources([ (vert.resources_required, vert.constraints) for vert in vertices ], chips) for (x, y, p, _, _), vert in zip(assigned_values, vertices): placement = Placement(vert, x, y, p) placements.add_placement(placement) else: (x, y, p, _, _) = resource_tracker.allocate_constrained_resources( vertex.resources_required, vertex.constraints, chips) placement = Placement(vertex, x, y, p) placements.add_placement(placement) return vertices
def check_types_of_edge_constraint(machine_graph): """ Go through the graph for operations and checks that the constraints\ are compatible. :param MachineGraph machine_graph: the graph to search through :raises PacmanConfigurationException: if a problem is found """ for partition in machine_graph.outgoing_edge_partitions: if partition.traffic_type != EdgeTrafficType.MULTICAST: continue fixed_key = locate_constraints_of_type( partition.constraints, FixedKeyAndMaskConstraint) fixed_mask = locate_constraints_of_type( partition.constraints, FixedMaskConstraint) fixed_field = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint) if len(fixed_key) > 1 or len(fixed_field) > 1 or len(fixed_mask) > 1: raise PacmanConfigurationException( "There are multiple constraint of the same type on partition " "{} starting at {}. Please fix and try again.".format( partition.identifier, partition.pre_vertex)) fixed_key = len(fixed_key) == 1 fixed_mask = len(fixed_mask) == 1 fixed_field = len(fixed_field) == 1 # check for fixed key and a fixed mask. as these should have been # merged before now if fixed_key and fixed_mask: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "mask constraint. These can be merged together, but is " "deemed an error here".format( partition.identifer, partition.pre_vertex)) # check for a fixed key and fixed field, as these are incompatible if fixed_key and fixed_field: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "field constraint. These may be merge-able together, but is " "deemed an error here".format( partition.identifer, partition.pre_vertex)) # check that a fixed mask and fixed field have compatible masks if fixed_mask and fixed_field: _check_masks_are_correct(partition)
def _compute_atoms_per_core(self, vertex, res_tracker, plan_n_timesteps): """ Work out how many atoms per core are required for the given\ vertex. Assumes that the first atom of the vertex is fully\ representative. :rtype: float """ # Get the usage of the first atom, then assume that this will be the # usage of all the atoms. requirements = vertex.get_resources_used_by_atoms(Slice(0, 1)) # Locate the maximum resources available limits = res_tracker.get_maximum_constrained_resources_available( requirements, vertex.constraints) # Find the ratio of each of the resources - if 0 is required, # assume the ratio is the max available atoms_per_sdram = self._get_ratio( limits.sdram.get_total_sdram(plan_n_timesteps), requirements.sdram.get_total_sdram(plan_n_timesteps)) atoms_per_dtcm = self._get_ratio( limits.dtcm.get_value(), requirements.dtcm.get_value()) atoms_per_cpu = self._get_ratio( limits.cpu_cycles.get_value(), requirements.cpu_cycles.get_value()) n_atoms = None for fa_constraint in utility_calls.locate_constraints_of_type( vertex.constraints, FixedVertexAtomsConstraint): if n_atoms is not None and n_atoms != fa_constraint.size: raise PacmanPartitionException( "Vertex has multiple contradictory fixed atom constraints" " - cannot be both {} and {}".format( n_atoms, fa_constraint.size)) n_atoms = fa_constraint.size max_atom_values = [atoms_per_sdram, atoms_per_dtcm, atoms_per_cpu] for max_atom_constraint in utility_calls.locate_constraints_of_type( vertex.constraints, MaxVertexAtomsConstraint): max_atom_values.append(float(max_atom_constraint.size)) max_atoms = min(max_atom_values) if n_atoms is not None and max_atoms < n_atoms: raise PacmanPartitionException( "Max size of {} is incompatible with fixed size of {}".format( max_atoms, n_atoms)) return n_atoms if n_atoms is not None else max_atoms
def _compute_atoms_per_core(self, vertex, res_tracker): """ Work out how many atoms per core are required for the given\ vertex. Assumes that the first atom of the vertex is fully\ representative. :rtype: float """ # Get the usage of the first atom, then assume that this will be the # usage of all the atoms. requirements = vertex.get_resources_used_by_atoms(Slice(0, 1)) # Locate the maximum resources available limits = res_tracker.get_maximum_constrained_resources_available( requirements, vertex.constraints) # Find the ratio of each of the resources - if 0 is required, # assume the ratio is the max available atoms_per_sdram = self._get_ratio(limits.sdram.get_value(), requirements.sdram.get_value()) atoms_per_dtcm = self._get_ratio(limits.dtcm.get_value(), requirements.dtcm.get_value()) atoms_per_cpu = self._get_ratio(limits.cpu_cycles.get_value(), requirements.cpu_cycles.get_value()) n_atoms = None for fa_constraint in utility_calls.locate_constraints_of_type( vertex.constraints, FixedVertexAtomsConstraint): if n_atoms is not None and n_atoms != fa_constraint.size: raise PacmanPartitionException( "Vertex has multiple contradictory fixed atom constraints" " - cannot be both {} and {}".format( n_atoms, fa_constraint.size)) n_atoms = fa_constraint.size max_atom_values = [atoms_per_sdram, atoms_per_dtcm, atoms_per_cpu] for max_atom_constraint in utility_calls.locate_constraints_of_type( vertex.constraints, MaxVertexAtomsConstraint): max_atom_values.append(float(max_atom_constraint.size)) max_atoms = min(max_atom_values) if n_atoms is not None and max_atoms < n_atoms: raise PacmanPartitionException( "Max size of {} is incompatible with fixed size of {}".format( max_atoms, n_atoms)) return n_atoms if n_atoms is not None else max_atoms
def _exists_equiv_vertex(x, y, graph, vertex_type): for vertex in graph.vertices: if isinstance(vertex, vertex_type) and any( constraint.x == x and constraint.y == y for constraint in locate_constraints_of_type( vertex.constraints, ChipAndCoreConstraint)): return vertex return None
def _check_if_partition_has_continuous_keys(partition): """ :param AbstractSingleSourcePartition partition: :rtype: bool """ continuous_constraints = locate_constraints_of_type( partition.constraints, ContiguousKeyRangeContraint) # TODO: Can we do better here? return len(continuous_constraints) > 0
def _determine_groups(self, machine_graph, graph_mapper, graph, n_keys_map, progress): check_types_of_edge_constraint(machine_graph) for partition in progress.over( machine_graph.outgoing_edge_partitions, False): fixed_key_constraints = locate_constraints_of_type( partition.constraints, FixedKeyAndMaskConstraint) fixed_mask_constraints = locate_constraints_of_type( partition.constraints, FixedMaskConstraint) fixed_field_constraints = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint) if (not fixed_key_constraints and not fixed_mask_constraints and not fixed_field_constraints): self.add_field_constraints( partition, graph_mapper, graph, n_keys_map)
def __call__(self, machine_graph, machine, plan_n_timesteps): """ :param machine_graph: The machine_graph to place :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param machine:\ The machine with respect to which to partition the application\ graph :type machine: :py:class:`spinn_machine.Machine` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :return: A set of placements :rtype: :py:class:`pacman.model.placements.Placements` :raise pacman.exceptions.PacmanPlaceException: \ If something goes wrong with the placement """ # check that the algorithm can handle the constraints self._check_constraints(machine_graph.vertices) # Sort the vertices into those with and those without # placement constraints placements = Placements() constrained = list() unconstrained = set() for vertex in machine_graph.vertices: if locate_constraints_of_type( vertex.constraints, AbstractPlacerConstraint): constrained.append(vertex) else: unconstrained.add(vertex) # Iterate over constrained vertices and generate placements progress = ProgressBar( machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) constrained = sort_vertices_by_known_constraints(constrained) for vertex in progress.over(constrained, False): self._place_vertex(vertex, resource_tracker, machine, placements) while unconstrained: # Place the subgraph with the overall most connected vertex max_connected_vertex = self._find_max_connected_vertex( unconstrained, machine_graph) self._place_unconstrained_subgraph( max_connected_vertex, machine_graph, unconstrained, machine, placements, resource_tracker, progress) # finished, so stop progress bar and return placements progress.end() return placements
def _run(self, machine_graph, machine, plan_n_timesteps): """ :param MachineGraph machine_graph: The machine_graph to place :param ~spinn_machine.Machine machine: The machine with respect to which to partition the application graph :param int plan_n_timesteps: number of timesteps to plan for :return: A set of placements :rtype: ~pacman.model.placements.Placements :raise PacmanPlaceException: If something goes wrong with the placement """ # check that the algorithm can handle the constraints self._check_constraints(machine_graph.vertices) # Sort the vertices into those with and those without # placement constraints placements = Placements() constrained = list() unconstrained = set() for vertex in machine_graph.vertices: if locate_constraints_of_type(vertex.constraints, AbstractPlacerConstraint): constrained.append(vertex) else: unconstrained.add(vertex) # Iterate over constrained vertices and generate placements progress = ProgressBar(machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) constrained = sort_vertices_by_known_constraints(constrained) vertices_on_same_chip = get_same_chip_vertex_groups(machine_graph) for vertex in progress.over(constrained, False): self._place_vertex(vertex, resource_tracker, machine, placements, vertices_on_same_chip, machine_graph) while unconstrained: # Place the subgraph with the overall most connected vertex max_connected_vertex = self._find_max_connected_vertex( unconstrained, machine_graph) self._place_unconstrained_subgraph(max_connected_vertex, machine_graph, unconstrained, machine, placements, resource_tracker, progress, vertices_on_same_chip) # finished, so stop progress bar and return placements progress.end() return placements
def _partition_vertex( self, vertex, subgraph, graph_to_subgraph_mapper, resource_tracker, graph): """ Partition a single vertex :param vertex: the vertex to partition :type vertex:\ :py:class:`pacman.model.partitionable_graph.abstract_partitionable_vertex.AbstractPartitionableVertex` :param subgraph: the partitioned_graph to add subverts to :type subgraph:\ py:class:`pacman.model.partitioned_graph.partitioned_graph.Subgraph` :param graph_to_subgraph_mapper: the mappings object from\ partitionable_graph to partitioned_graph which needs to be\ updated with new subverts :type graph_to_subgraph_mapper:\ py:class:'pacman.modelgraph_subgraph_mapper.graph_mapper.GraphMapper' :param resource_tracker: A tracker of assigned resources :type resource_tracker:\ :py:class:`pacman.utilities.resource_tracker.ResourceTracker` :param graph: the partitionable_graph object :type graph:\ :py:class:`pacman.model.graph.partitionable_graph.PartitionableGraph` :return: None :rtype: None :raise pacman.exceptions.PacmanPartitionException: if the extra vertex\ for partitioning identically has a different number of\ atoms than its counterpart. """ partiton_together_vertices = \ self._locate_vertices_to_partition_now(vertex) # locate max atoms per core possible_max_atoms = list() possible_max_atoms.append(vertex.get_max_atoms_per_core()) for other_partitionable_vertex in partiton_together_vertices: max_atom_constraints =\ utility_calls.locate_constraints_of_type( other_partitionable_vertex.constraints, PartitionerMaximumSizeConstraint) for constraint in max_atom_constraints: possible_max_atoms.append(constraint.size) max_atoms_per_core = min(possible_max_atoms) # partition by atoms self._partition_by_atoms( partiton_together_vertices, vertex.n_atoms, max_atoms_per_core, subgraph, graph, graph_to_subgraph_mapper, resource_tracker)
def _partition_vertex(self, vertex, subgraph, graph_to_subgraph_mapper, resource_tracker, graph): """ Partition a single vertex :param vertex: the vertex to partition :type vertex:\ :py:class:`pacman.model.partitionable_graph.abstract_partitionable_vertex.AbstractPartitionableVertex` :param subgraph: the partitioned_graph to add subverts to :type subgraph:\ py:class:`pacman.model.partitioned_graph.partitioned_graph.Subgraph` :param graph_to_subgraph_mapper: the mappings object from\ partitionable_graph to partitioned_graph which needs to be\ updated with new subverts :type graph_to_subgraph_mapper:\ py:class:'pacman.modelgraph_subgraph_mapper.graph_mapper.GraphMapper' :param resource_tracker: A tracker of assigned resources :type resource_tracker:\ :py:class:`pacman.utilities.resource_tracker.ResourceTracker` :param graph: the partitionable_graph object :type graph:\ :py:class:`pacman.model.graph.partitionable_graph.PartitionableGraph` :return: None :rtype: None :raise pacman.exceptions.PacmanPartitionException: if the extra vertex\ for partitioning identically has a different number of\ atoms than its counterpart. """ partiton_together_vertices = \ self._locate_vertices_to_partition_now(vertex) # locate max atoms per core possible_max_atoms = list() possible_max_atoms.append(vertex.get_max_atoms_per_core()) for other_partitionable_vertex in partiton_together_vertices: max_atom_constraints =\ utility_calls.locate_constraints_of_type( other_partitionable_vertex.constraints, PartitionerMaximumSizeConstraint) for constraint in max_atom_constraints: possible_max_atoms.append(constraint.size) max_atoms_per_core = min(possible_max_atoms) # partition by atoms self._partition_by_atoms(partiton_together_vertices, vertex.n_atoms, max_atoms_per_core, subgraph, graph, graph_to_subgraph_mapper, resource_tracker)
def _sort_vertices_for_one_to_one_connection(self, partitioned_graph): """ :param partitioned_graph: the partitioned graph of this application :return: list of sorted vertices """ sorted_vertices = list() found_list = list() # order subverts based on constraint priority ordered_subverts = utility_calls.sort_objects_by_constraint_authority( partitioned_graph.subvertices) for vertex in ordered_subverts: incoming_edges = \ partitioned_graph.incoming_subedges_from_subvertex(vertex) # do search if not already added and has incoming edges if vertex not in found_list and len(incoming_edges) != 0: chip_constraint = utility_calls.locate_constraints_of_type( vertex.constraints, PlacerChipAndCoreConstraint) # if has constraint, add first then add incoming if len(chip_constraint) != 0: one_to_one_incoming_edges = list() one_to_one_incoming_edges.append(vertex) sorted_vertices.append(one_to_one_incoming_edges) found_list.append(vertex) self.check_incoming_verts(one_to_one_incoming_edges, vertex, partitioned_graph, found_list) else: # if no constraint add incoming then first one_to_one_incoming_edges = list() sorted_vertices.append(one_to_one_incoming_edges) self.check_incoming_verts(one_to_one_incoming_edges, vertex, partitioned_graph, found_list) one_to_one_incoming_edges.append(vertex) found_list.append(vertex) # locate vertices which have no output or input, and add them for # placement for vertex in ordered_subverts: if vertex not in found_list: listed_vertex = list() listed_vertex.append(vertex) sorted_vertices.append(listed_vertex) return sorted_vertices
def is_contiguous_range(same_key_group): """ Determine if any edge in the group has a\ :py:class:`pacman.model.constraints.key_allocator_contiguous_range_constraint.KeyAllocatorContiguousRangeContraint` :param same_key_group: Set of partitioned edges that are to be\ assigned the same keys and masks :type same_key_group: iterable of\ :py:class:`pacman.model.partitioned_graph.partitioned_edge.PartitionedEdge` :return: True if the range should be contiguous """ for edge in same_key_group: constraints = utility_calls.locate_constraints_of_type( edge.constraints, KeyAllocatorContiguousRangeContraint) if len(constraints) > 0: return True return False
def _get_key_constraints(outgoing_partition): """ locate :param outgoing_partition: the outgoing partition to find constraints for :return: the nengo key constraints holder or None if none exist """ if isinstance( outgoing_partition.pre_vertex, AbstractNengoMachineVertex): outgoing_partition_constraints = \ outgoing_partition.pre_vertex.\ get_outgoing_partition_constraints(outgoing_partition) return utility_calls.locate_constraints_of_type( constraints=outgoing_partition_constraints, constraint_type=NengoKeyConstraints)[0] else: raise Exception( "this outgoing partition has no nengo key constraints. " "Dont know how to handle this")
def _locate_connected_chip_data(vertex, machine): """ Finds the connected virtual chip :param vertex: :param machine: """ # locate the chip from the placement constraint placement_constraints = locate_constraints_of_type( vertex.constraints, ChipAndCoreConstraint) routers = ( machine.get_chip_at(constraint.x, constraint.y).router for constraint in placement_constraints) links = ( router.get_link(i) for router in routers for i in range(6) if router.is_link(i)) link = next(iter(links), None) if link is None: raise PacmanConfigurationException( "Can't find the real chip this virtual chip is connected to." "Please fix and try again.") return ([link.destination_x, link.destination_y], link.multicast_default_from)
def _locate_connected_chip_data(vertex, machine): """ Finds the connected virtual chip :param vertex: :param machine: """ # locate the chip from the placement constraint placement_constraints = utility_calls.locate_constraints_of_type( vertex.constraints, ChipAndCoreConstraint) routers = ( machine.get_chip_at(constraint.x, constraint.y).router for constraint in placement_constraints) links = ( router.get_link(i) for router in routers for i in range(6) if router.is_link(i)) link = next(iter(links), None) if link is None: raise PacmanConfigurationException( "Can't find the real chip this virtual chip is connected to." "Please fix and try again.") return ([link.destination_x, link.destination_y], link.multicast_default_from)
def get_fixed_mask(same_key_group): """ Get a fixed mask from a group of partitioned edges if a\ :py:class:`pacman.model.constraints.key_allocator_same_key_constraint.KeyAllocatorFixedMaskConstraint`\ constraint exists in any of the edges in the group. :param same_key_group: Set of partitioned edges that are to be\ assigned the same keys and masks :type same_key_group: iterable of\ :py:class:`pacman.model.partitioned_graph.partitioned_edge.PartitionedEdge` :return: The fixed mask if found, or None :raise PacmanValueError: If two edges conflict in their requirements """ mask = None fields = None edge_with_mask = None for edge in same_key_group: fixed_mask_constraints = utility_calls.locate_constraints_of_type( edge.constraints, KeyAllocatorFixedMaskConstraint) for fixed_mask_constraint in fixed_mask_constraints: if mask is not None and mask != fixed_mask_constraint.mask: raise PacmanValueError( "Two Partitioned Edges {} and {} must have the same" " key and mask, but have different fixed masks," " {} and {}".format(edge, edge_with_mask, mask, fixed_mask_constraint.mask)) if (fields is not None and fixed_mask_constraint.fields is not None and fields != fixed_mask_constraint.fields): raise PacmanValueError( "Two Partitioned Edges {} and {} must have the same" " key and mask, but have different field ranges" .format(edge, edge_with_mask)) mask = fixed_mask_constraint.mask edge_with_mask = edge if fixed_mask_constraint.fields is not None: fields = fixed_mask_constraint.fields return mask, fields
def __call__(self, machine_graph, machine): # check that the algorithm can handle the constraints self._check_constraints(machine_graph.vertices) # Sort the vertices into those with and those without # placement constraints placements = Placements() constrained = list() unconstrained = set() for vertex in machine_graph.vertices: if locate_constraints_of_type(vertex.constraints, AbstractPlacerConstraint): constrained.append(vertex) else: unconstrained.add(vertex) # Iterate over constrained vertices and generate placements progress = ProgressBar(machine_graph.n_vertices, "Placing graph vertices") resource_tracker = ResourceTracker( machine, self._generate_radial_chips(machine)) constrained = sort_vertices_by_known_constraints(constrained) for vertex in progress.over(constrained, False): self._place_vertex(vertex, resource_tracker, machine, placements) while unconstrained: # Place the subgraph with the overall most connected vertex max_connected_vertex = self._find_max_connected_vertex( unconstrained, machine_graph) self._place_unconstrained_subgraph(max_connected_vertex, machine_graph, unconstrained, machine, placements, resource_tracker, progress) # finished, so stop progress bar and return placements progress.end() return placements
def get_fixed_mask(same_key_group): """ Get a fixed mask from a group of edges if a\ :py:class:`pacman.model.constraints.key_allocator_constraints.FixedMaskConstraint`\ constraint exists in any of the edges in the group. :param same_key_group: \ Set of edges that are to be assigned the same keys and masks :type same_key_group: \ iterable(:py:class:`pacman.model.graphs.machine.MachineEdge`) :return: The fixed mask if found, or None :raise PacmanValueError: If two edges conflict in their requirements """ mask = None fields = None edge_with_mask = None for edge in same_key_group: fixed_mask_constraints = locate_constraints_of_type( edge.constraints, FixedMaskConstraint) for fixed_mask_constraint in fixed_mask_constraints: if mask is not None and mask != fixed_mask_constraint.mask: raise PacmanValueError( "Two Edges {} and {} must have the same" " key and mask, but have different fixed masks," " {} and {}".format(edge, edge_with_mask, mask, fixed_mask_constraint.mask)) if (fields is not None and fixed_mask_constraint.fields is not None and fields != fixed_mask_constraint.fields): raise PacmanValueError( "Two Edges {} and {} must have the same" " key and mask, but have different field ranges".format( edge, edge_with_mask)) mask = fixed_mask_constraint.mask edge_with_mask = edge if fixed_mask_constraint.fields is not None: fields = fixed_mask_constraint.fields return mask, fields
def _extract_keys_and_masks_from_bit_field( self, partition, bit_field_space, n_keys_map, seen_mask_instances): """ Take the bit field space and a partition and locate the keys and\ masks for that partition given all its constraints :param partition: the partition to extract keys and masks for :param bit_field_space: the bit field space :param n_keys_map: the edge to n_keys map :param seen_mask_instances: a count of how many fixed mask instances\ have been seen, so that it can increment the routing part of the\ fixed mask fields. :return: the routing keys and masks for the partition and the number\ of seen fixed masks instances """ routing_keys_and_masks = list() application_keys_and_masks = list() fixed_key_constraints = locate_constraints_of_type( partition.constraints, FixedKeyAndMaskConstraint) fixed_mask_constraints = locate_constraints_of_type( partition.constraints, FixedMaskConstraint) fixed_field_constraints = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint) flexi_field_constraints = locate_constraints_of_type( partition.constraints, FlexiKeyFieldConstraint) continuous_constraints = locate_constraints_of_type( partition.constraints, ContiguousKeyRangeContraint) if fixed_key_constraints: fixed_keys_fields = \ self._field_mapper[fixed_key_constraints[0].keys_and_masks[0]] # tracker for iterator fields range_based_fixed_key_fields = list() # add the app field (must exist in this situation) inputs = dict() inputs[APPLICATION_DIVIDER_FIELD_NAME] = \ self._fixed_key_application_field_value # handle the inputs for static parts of the mask for fixed_key_field in fixed_keys_fields: app_field_space = bit_field_space(**inputs) tag = list(app_field_space.get_tags( str(fixed_key_field.name)))[0] if tag == SUPPORTED_TAGS.ROUTING.name: inputs[str(fixed_key_field.name)] = fixed_key_field.value elif tag == SUPPORTED_TAGS.APPLICATION.name: range_based_fixed_key_fields.append(fixed_key_field) else: raise exceptions.PacmanConfigurationException( "Don't know this tag field, sorry") if len(range_based_fixed_key_fields) > 1: raise exceptions.PacmanConfigurationException( "Multiple fixed key fields are not supported") # get n keys from n_keys_map for the range based mask part n_keys = n_keys_map.n_keys_for_partition(partition) # generate keys for key_index in range(n_keys): inputs[str(range_based_fixed_key_fields[0].name)] = key_index # routing keys and masks routing_key = bit_field_space(**inputs).get_value( tag=SUPPORTED_TAGS.ROUTING.name) routing_mask = bit_field_space(**inputs).get_mask( tag=SUPPORTED_TAGS.ROUTING.name) routing_keys_and_masks.append(BaseKeyAndMask(routing_key, routing_mask)) # application keys and masks application_key = bit_field_space(**inputs).get_value( tag=SUPPORTED_TAGS.APPLICATION.name) application_mask = bit_field_space(**inputs).get_mask( tag=SUPPORTED_TAGS.APPLICATION.name) application_keys_and_masks.append(BaseKeyAndMask( application_key, application_mask)) elif fixed_mask_constraints: # get constraint and its fields fixed_mask_constraint_mask = fixed_mask_constraints[0].mask fixed_mask_fields = self._field_mapper[fixed_mask_constraint_mask] # tracker for iterator fields range_based_fixed_mask_fields = list() # add the app field (must exist in this situation) inputs = dict() inputs[APPLICATION_DIVIDER_FIELD_NAME] = \ self._fixed_mask_application_field_value # handle the inputs for static parts of the mask for fixed_mask_field in fixed_mask_fields: app_field_space = bit_field_space(**inputs) tag = list(app_field_space.get_tags(str(fixed_mask_field)))[0] if tag == SUPPORTED_TAGS.ROUTING.name: inputs[str(fixed_mask_field)] = seen_mask_instances elif tag == SUPPORTED_TAGS.APPLICATION.name: range_based_fixed_mask_fields.append(fixed_mask_field) else: raise exceptions.PacmanConfigurationException( "I don't recognise this tag. sorry") if len(range_based_fixed_mask_fields) > 1: raise exceptions.PacmanConfigurationException( "Multiple fixed mask fields are not supported") # get n keys from n_keys_map for the range based mask part n_keys = n_keys_map.n_keys_for_partition(partition) # generate keys for key_index in range(n_keys): inputs[str(range_based_fixed_mask_fields[0])] = key_index # routing keys and masks routing_key = bit_field_space(**inputs).get_value( tag=SUPPORTED_TAGS.ROUTING.name) routing_mask = bit_field_space(**inputs).get_mask( tag=SUPPORTED_TAGS.ROUTING.name) routing_keys_and_masks.append(BaseKeyAndMask(routing_key, routing_mask)) # application keys and masks application_key = bit_field_space(**inputs).get_value( tag=SUPPORTED_TAGS.APPLICATION.name) application_mask = bit_field_space(**inputs).get_mask( tag=SUPPORTED_TAGS.APPLICATION.name) application_keys_and_masks.append(BaseKeyAndMask( application_key, application_mask)) # update the seen mask instances so the next one gets a new key seen_mask_instances += 1 elif fixed_field_constraints: # TODO: need to fill this out raise exceptions.PacmanConfigurationException( "Fixed field constraints are not supported") elif flexi_field_constraints: inputs = dict() # if there's a application field, add the value if self._flexi_field_application_field_values: inputs[APPLICATION_DIVIDER_FIELD_NAME] = \ self._flexi_field_application_field_values[ flexi_field_constraints[0].fields[0].name] # collect flexible fields by group range_based_fixed_mask_fields = list() for field in flexi_field_constraints[0].fields: if field.value is not None: inputs[field.name] = field.value else: range_based_fixed_mask_fields.append(field) # if the set contains a range_based flexible field do search if range_based_fixed_mask_fields: routing_keys_and_masks, application_keys_and_masks = \ self._handle_set_of_flexi_range_fields( range_based_fixed_mask_fields, bit_field_space, routing_keys_and_masks, application_keys_and_masks, inputs, 0) else: # no range, just grab the key and value key = bit_field_space(**inputs).get_value() mask = bit_field_space(**inputs).get_mask() routing_keys_and_masks.append(BaseKeyAndMask(key, mask)) # if there's a continuous constraint check, check the keys for # continuity if continuous_constraints: if not self._check_keys_are_continuous( application_keys_and_masks): raise exceptions.PacmanConfigurationException( "These keys returned from the bitfield are" "not continuous. Therefore cannot be used") # if continuous, we only need the first key, so drop the rest routing_keys_and_masks = routing_keys_and_masks[0:1] # return keys and masks return routing_keys_and_masks, seen_mask_instances
def _check_if_partition_has_continuous_keys(partition): continuous_constraints = utility_calls.locate_constraints_of_type( partition.constraints, ContiguousKeyRangeContraint) # TODO: Can we do better here? return len(continuous_constraints) > 0
def _do_allocation( self, one_to_one_groups, same_chip_vertex_groups, machine, plan_n_timesteps, machine_graph, progress): """ :param one_to_one_groups: Groups of vertexes that would be nice on same chip :type one_to_one_groups: list(set(vertex)) :param same_chip_vertex_groups: Mapping of Vertex to the Vertex that must be on the same Chip :type same_chip_vertex_groups: dict(vertex, collection(vertex)) :param machine:\ The machine with respect to which to partition the application\ graph :type machine: :py:class:`spinn_machine.Machine` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :param machine_graph: The machine_graph to place :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param progress: :return: """ placements = Placements() resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) all_vertices_placed = set() # RadialPlacementFromChipConstraint won't work here for vertex in machine_graph.vertices: for constraint in vertex.constraints: if isinstance(constraint, RadialPlacementFromChipConstraint): raise PacmanPlaceException( "A RadialPlacementFromChipConstraint will not work " "with the OneToOnePlacer algorithm; use the " "RadialPlacer algorithm instead") unconstrained = list() # Find and place vertices with hard constraints for vertex in machine_graph.vertices: if isinstance(vertex, AbstractVirtualVertex): virtual_p = 0 while placements.is_processor_occupied( vertex.virtual_chip_x, vertex.virtual_chip_y, virtual_p): virtual_p += 1 placements.add_placement(Placement( vertex, vertex.virtual_chip_x, vertex.virtual_chip_y, virtual_p)) all_vertices_placed.add(vertex) elif locate_constraints_of_type( vertex.constraints, ChipAndCoreConstraint): self._allocate_same_chip_as_group( vertex, placements, resource_tracker, same_chip_vertex_groups, all_vertices_placed, progress) else: unconstrained.append(vertex) for grouped_vertices in one_to_one_groups: # Get unallocated vertices and placements of allocated vertices unallocated = list() chips = list() for vert in grouped_vertices: if vert in all_vertices_placed: placement = placements.get_placement_of_vertex(vert) chips.append((placement.x, placement.y)) else: unallocated.append(vert) if 0 < len(unallocated) <=\ resource_tracker.get_maximum_cores_available_on_a_chip(): # Try to allocate all vertices to the same chip self._allocate_one_to_one_group( resource_tracker, unallocated, progress, placements, chips, all_vertices_placed) # if too big or failed go on to other groups first # check all have been allocated if not do so now. for vertex in machine_graph.vertices: if vertex not in all_vertices_placed: self._allocate_same_chip_as_group( vertex, placements, resource_tracker, same_chip_vertex_groups, all_vertices_placed, progress) progress.end() return placements
def __call__(self, subgraph, n_keys_map, graph_mapper=None): # check that this algorithm supports the constraints utility_calls.check_algorithm_can_support_constraints( constrained_vertices=subgraph.partitions, supported_constraints=[ KeyAllocatorFixedMaskConstraint, KeyAllocatorFixedKeyAndMaskConstraint, KeyAllocatorContiguousRangeContraint ], abstract_constraint_type=AbstractKeyAllocatorConstraint) # verify that no edge has more than 1 of a constraint ,and that # constraints are compatible routing_info_allocator_utilities.\ check_types_of_edge_constraint(subgraph) routing_infos = RoutingInfo() # Get the partitioned edges grouped by those that require the same key (fixed_key_groups, fixed_mask_groups, fixed_field_groups, flexi_field_groups, continuous_groups, none_continuous_groups) = \ routing_info_allocator_utilities.get_edge_groups(subgraph) # Even non-continuous keys will be continuous for group in none_continuous_groups: continuous_groups.add(group) # Go through the groups and allocate keys progress_bar = ProgressBar(len(subgraph.partitions), "Allocating routing keys") # allocate the groups that have fixed keys for group in fixed_key_groups: # fixed keys groups # Get any fixed keys and masks from the group and attempt to # allocate them fixed_mask = None fixed_key_and_mask_constraint = \ utility_calls.locate_constraints_of_type( group.constraints, KeyAllocatorFixedKeyAndMaskConstraint)[0] # attempt to allocate them self._allocate_fixed_keys_and_masks( fixed_key_and_mask_constraint.keys_and_masks, fixed_mask) # update the pacman data objects self._update_routing_objects( fixed_key_and_mask_constraint.keys_and_masks, routing_infos, group) continuous_groups.remove(group) progress_bar.update() for group in fixed_mask_groups: # fixed mask groups # get mask and fields if need be fixed_mask = utility_calls.locate_constraints_of_type( group.constraints, KeyAllocatorFixedMaskConstraint)[0].mask fields = None if group in fixed_field_groups: fields = utility_calls.locate_constraints_of_type( group.constraints, KeyAllocatorFixedFieldConstraint)[0].fields fixed_field_groups.remove(group) # try to allocate keys_and_masks = self._allocate_keys_and_masks( fixed_mask, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous_groups.remove(group) progress_bar.update() for group in fixed_field_groups: fields = utility_calls.locate_constraints_of_type( group.constraints, KeyAllocatorFixedFieldConstraint)[0].fields # try to allocate keys_and_masks = self._allocate_keys_and_masks( None, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous_groups.remove(group) progress_bar.update() if len(flexi_field_groups) != 0: raise exceptions.PacmanConfigurationException( "MallocBasedRoutingInfoAllocator does not support FlexiField") # If there is a graph, group by source vertex and sort by vertex slice # (lo_atom) if graph_mapper is not None: vertex_groups = defaultdict(list) for partition in continuous_groups: vertex = graph_mapper.get_vertex_from_subvertex( partition.edges[0].pre_subvertex) vertex_groups[vertex].append(partition) vertex_partitions = list() for vertex_group in vertex_groups.itervalues(): sorted_partitions = sorted( vertex_group, key=lambda part: graph_mapper.get_subvertex_slice( part.edges[0].pre_subvertex)) vertex_partitions.extend(sorted_partitions) continuous_groups = vertex_partitions for group in continuous_groups: keys_and_masks = self._allocate_keys_and_masks( None, None, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) progress_bar.end() return {'routing_infos': routing_infos}
def __call__(self, partitioned_graph, file_path): """ :param partitioned_graph: :param folder_path: :return: """ progress_bar = ProgressBar(len(partitioned_graph.subvertices), "Converting to json partitioned graph") # write basic stuff json_graph_dictory_rep = dict() # write vertices data vertices_resources = dict() json_graph_dictory_rep["vertices_resources"] = vertices_resources edges_resources = defaultdict() json_graph_dictory_rep["edges"] = edges_resources for vertex in partitioned_graph.subvertices: # handle external devices if isinstance(vertex, AbstractVirtualVertex): vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = 0 # handle tagged vertices elif len(utility_calls.locate_constraints_of_type(vertex.constraints, AbstractTagAllocatorConstraint)) != 0: # handle the edge between the tag-able vertex and the fake # vertex hyper_edge_dict = dict() edges_resources[hashlib.md5(vertex.label).hexdigest()] = hyper_edge_dict hyper_edge_dict["source"] = str(id(vertex)) hyper_edge_dict["sinks"] = [hashlib.md5(vertex.label).hexdigest()] hyper_edge_dict["weight"] = 1.0 hyper_edge_dict["type"] = "FAKE_TAG_EDGE" # add the tag-able vertex vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = DEFAULT_NOUMBER_OF_CORES_USED_PER_PARTITIONED_VERTEX vertex_resources["sdram"] = int(vertex.resources_required.sdram.get_value()) # add fake vertex vertex_resources = dict() vertices_resources[hashlib.md5(vertex.label).hexdigest()] = vertex_resources vertex_resources["cores"] = 0 vertex_resources["sdram"] = 0 # handle standard vertices else: vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = DEFAULT_NOUMBER_OF_CORES_USED_PER_PARTITIONED_VERTEX vertex_resources["sdram"] = int(vertex.resources_required.sdram.get_value()) vertex_outgoing_partitions = partitioned_graph.outgoing_edges_partitions_from_vertex(vertex) # handle the vertex edges for vertex_partition in vertex_outgoing_partitions: hyper_edge_dict = dict() edges_resources["{}:{}".format(id(vertex), vertex_partition)] = hyper_edge_dict hyper_edge_dict["source"] = str(id(vertex)) sinks_string = [] for edge in vertex_outgoing_partitions[vertex_partition].edges: sinks_string.append(str(id(edge.post_subvertex))) hyper_edge_dict["sinks"] = sinks_string hyper_edge_dict["weight"] = 1.0 hyper_edge_dict["type"] = vertex_outgoing_partitions[vertex_partition].type.name.lower() progress_bar.update() file_to_write = open(file_path, "w") json.dump(json_graph_dictory_rep, file_to_write) file_to_write.close() # validate the schema partitioned_graph_schema_file_path = os.path.join( os.path.dirname(file_format_schemas.__file__), "partitioned_graph.json" ) file_to_read = open(partitioned_graph_schema_file_path, "r") partitioned_graph_schema = json.load(file_to_read) jsonschema.validate(json_graph_dictory_rep, partitioned_graph_schema) progress_bar.end() return {"partitioned_graph": file_path}
def _check_if_partition_has_continuous_keys(partition): continuous_constraints =\ utility_calls.locate_constraints_of_type( partition.constraints, KeyAllocatorContiguousRangeContraint) return not (len(continuous_constraints) == 0)
def get_edge_groups(machine_graph, traffic_type): """ Utility method to get groups of edges using any\ :py:class:`pacman.model.constraints.key_allocator_constraints.KeyAllocatorSameKeyConstraint`\ constraints. Note that no checking is done here about conflicts\ related to other constraints. :param machine_graph: the machine graph :param traffic_type: the traffic type to group """ # mapping between partition and shared key group it is in partition_groups = dict() # process each partition one by one in a bubble sort kinda way for vertex in machine_graph.vertices: for partition in machine_graph.\ get_outgoing_edge_partitions_starting_at_vertex(vertex): # only process partitions of the correct traffic type if partition.traffic_type == traffic_type: # Get a set of partitions that should be grouped together shared_key_constraints = locate_constraints_of_type( partition.constraints, ShareKeyConstraint) partitions_to_group = [partition] for constraint in shared_key_constraints: partitions_to_group.extend(constraint.other_partitions) # Get a set of groups that should be grouped groups_to_group = [ partition_groups.get(part_to_group, [part_to_group]) for part_to_group in partitions_to_group ] # Group the groups new_group = ConstraintGroup(part for group in groups_to_group for part in group) partition_groups.update( {part: new_group for part in new_group}) # Keep track of groups fixed_key_groups = list() shared_key_groups = list() fixed_mask_groups = list() fixed_field_groups = list() flexi_field_groups = list() continuous_groups = list() noncontinuous_groups = list() groups_by_type = { FixedKeyAndMaskConstraint: fixed_key_groups, FixedMaskConstraint: fixed_mask_groups, FixedKeyFieldConstraint: fixed_field_groups, FlexiKeyFieldConstraint: flexi_field_groups, } groups = set(itervalues(partition_groups)) for group in groups: # Get all expected constraints in the group constraints = [ constraint for partition in group for constraint in locate_constraints_of_type( partition.constraints, (FixedKeyAndMaskConstraint, FixedMaskConstraint, FlexiKeyFieldConstraint, FixedKeyFieldConstraint)) ] # Check that the possibly conflicting constraints are equal if constraints and not all(constraint_a == constraint_b for constraint_a in constraints for constraint_b in constraints): raise PacmanRouteInfoAllocationException( "The group of partitions {} have conflicting constraints". format(constraints)) # If no constraints, must be one of the non-specific groups if not constraints: # If the group has only one item, it is not shared if len(group) == 1: continuous_constraints = [ constraint for partition in group for constraint in locate_constraints_of_type( constraints, ContiguousKeyRangeContraint) ] if continuous_constraints: continuous_groups.append(group) else: noncontinuous_groups.append(group) # If the group has more than one partition, it must be shared else: shared_key_groups.append(group) # If constraints found, put the group in the appropriate constraint # group else: group._set_constraint(constraints[0]) constraint_type = type(constraints[0]) groups_by_type[constraint_type].append(group) # return the set of groups return (fixed_key_groups, shared_key_groups, fixed_mask_groups, fixed_field_groups, flexi_field_groups, continuous_groups, noncontinuous_groups)
def check_types_of_edge_constraint(machine_graph): """ Go through the graph for operations and checks that the constraints\ are compatible. :param machine_graph: the graph to search through :rtype: None: """ for partition in machine_graph.outgoing_edge_partitions: fixed_key = locate_constraints_of_type(partition.constraints, FixedKeyAndMaskConstraint) fixed_mask = locate_constraints_of_type(partition.constraints, FixedMaskConstraint) fixed_field = locate_constraints_of_type(partition.constraints, FixedKeyFieldConstraint) flexi_field = locate_constraints_of_type(partition.constraints, FlexiKeyFieldConstraint) if (len(fixed_key) > 1 or len(fixed_field) > 1 or len(fixed_mask) > 1 or len(flexi_field) > 1): raise PacmanConfigurationException( "There are more than one of the same constraint type on " "the partition {} starting at {}. Please fix and try again.". format(partition.identifier, partition.pre_vertex)) fixed_key = len(fixed_key) == 1 fixed_mask = len(fixed_mask) == 1 fixed_field = len(fixed_field) == 1 flexi_field = len(flexi_field) == 1 # check for fixed key and a fixed mask. as these should have been # merged before now if fixed_key and fixed_mask: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "mask constraint. These can be merged together, but is " "deemed an error here".format(partition.identifer, partition.pre_vertex)) # check for a fixed key and fixed field, as these are incompatible if fixed_key and fixed_field: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "field constraint. These may be merge-able together, but " "is deemed an error here".format(partition.identifer, partition.pre_vertex)) # check that a fixed mask and fixed field have compatible masks if fixed_mask and fixed_field: _check_masks_are_correct(partition) # check that if there's a flexible field, and something else, throw # error if flexi_field and (fixed_mask or fixed_key or fixed_field): raise PacmanConfigurationException( "The partition {} starting at {} has a flexible field and " "another fixed constraint. These maybe be merge-able, but " "is deemed an error here".format(partition.identifer, partition.pre_vertex))
def get_edge_groups(machine_graph, traffic_type): """ Utility method to get groups of edges using any\ :py:class:`~pacman.model.constraints.key_allocator_constraints.KeyAllocatorSameKeyConstraint`\ constraints. Note that no checking is done here about conflicts\ related to other constraints. :param machine_graph: the machine graph :param traffic_type: the traffic type to group """ # mapping between partition and shared key group it is in partition_groups = OrderedDict() # process each partition one by one in a bubble sort kinda way for vertex in machine_graph.vertices: for partition in machine_graph.\ get_outgoing_edge_partitions_starting_at_vertex(vertex): # only process partitions of the correct traffic type if partition.traffic_type == traffic_type: # Get a set of partitions that should be grouped together shared_key_constraints = locate_constraints_of_type( partition.constraints, ShareKeyConstraint) partitions_to_group = [partition] for constraint in shared_key_constraints: partitions_to_group.extend(constraint.other_partitions) # Get a set of groups that should be grouped groups_to_group = [ partition_groups.get(part_to_group, [part_to_group]) for part_to_group in partitions_to_group] # Group the groups new_group = ConstraintGroup( part for group in groups_to_group for part in group) partition_groups.update( {part: new_group for part in new_group}) # Keep track of groups fixed_key_groups = list() shared_key_groups = list() fixed_mask_groups = list() fixed_field_groups = list() flexi_field_groups = list() continuous_groups = list() noncontinuous_groups = list() groups_by_type = { FixedKeyAndMaskConstraint: fixed_key_groups, FixedMaskConstraint: fixed_mask_groups, FixedKeyFieldConstraint: fixed_field_groups, FlexiKeyFieldConstraint: flexi_field_groups, } groups = OrderedSet(itervalues(partition_groups)) for group in groups: # Get all expected constraints in the group constraints = [ constraint for partition in group for constraint in locate_constraints_of_type( partition.constraints, (FixedKeyAndMaskConstraint, FixedMaskConstraint, FlexiKeyFieldConstraint, FixedKeyFieldConstraint))] # Check that the possibly conflicting constraints are equal if constraints and not all( constraint_a == constraint_b for constraint_a in constraints for constraint_b in constraints): raise PacmanRouteInfoAllocationException( "The group of partitions {} have conflicting constraints" .format(constraints)) # If no constraints, must be one of the non-specific groups if not constraints: # If the group has only one item, it is not shared if len(group) == 1: continuous_constraints = [ constraint for partition in group for constraint in locate_constraints_of_type( constraints, ContiguousKeyRangeContraint)] if continuous_constraints: continuous_groups.append(group) else: noncontinuous_groups.append(group) # If the group has more than one partition, it must be shared else: shared_key_groups.append(group) # If constraints found, put the group in the appropriate constraint # group else: group._set_constraint(constraints[0]) constraint_type = type(constraints[0]) groups_by_type[constraint_type].append(group) # return the set of groups return (fixed_key_groups, shared_key_groups, fixed_mask_groups, fixed_field_groups, flexi_field_groups, continuous_groups, noncontinuous_groups)
def __call__(self, partitioned_graph, machine): # check that the algorithm can handle the constraints utility_calls.check_algorithm_can_support_constraints( constrained_vertices=partitioned_graph.subvertices, supported_constraints=[ PlacerRadialPlacementFromChipConstraint, TagAllocatorRequireIptagConstraint, TagAllocatorRequireReverseIptagConstraint, PlacerChipAndCoreConstraint], abstract_constraint_type=AbstractPlacerConstraint) # Sort the vertices into those with and those without # placement constraints placements = Placements() constrained_vertices = list() unconstrained_vertices = set() for subvertex in partitioned_graph.subvertices: placement_constraints = utility_calls.locate_constraints_of_type( subvertex.constraints, AbstractPlacerConstraint) if len(placement_constraints) > 0: constrained_vertices.append(subvertex) else: unconstrained_vertices.add(subvertex) # Iterate over constrained vertices and generate placements progress_bar = ProgressBar(len(partitioned_graph.subvertices), "Placing graph vertices") resource_tracker = ResourceTracker( machine, self._generate_radial_chips(machine)) for vertex in constrained_vertices: self._place_vertex(vertex, resource_tracker, machine, placements) progress_bar.update() while len(unconstrained_vertices) > 0: # Keep track of all vertices connected to the currently placed ones next_vertices = set() # Initially, add the overall most connected vertex max_connected_vertex = self._find_max_connected_vertex( unconstrained_vertices, partitioned_graph) next_vertices.add(max_connected_vertex) while len(next_vertices) > 0: # Find the vertex most connected to the currently placed # vertices vertex = self._find_max_connected_vertex(next_vertices, partitioned_graph) # Place the vertex self._place_vertex(vertex, resource_tracker, machine, placements) progress_bar.update() unconstrained_vertices.remove(vertex) next_vertices.remove(vertex) # Add all vertices connected to this one to the set for in_edge in (partitioned_graph .incoming_subedges_from_subvertex(vertex)): if in_edge.pre_subvertex in unconstrained_vertices: next_vertices.add(in_edge.pre_subvertex) for out_edge in (partitioned_graph .outgoing_subedges_from_subvertex(vertex)): if out_edge.post_subvertex in unconstrained_vertices: next_vertices.add(out_edge.post_subvertex) # finished, so stop progress bar and return placements progress_bar.end() return {'placements': placements}
def _do_allocation( self, one_to_one_groups, same_chip_vertex_groups, machine, plan_n_timesteps, machine_graph, progress): """ :param list(set(MachineVertex)) one_to_one_groups: Groups of vertexes that would be nice on same chip :param same_chip_vertex_groups: Mapping of Vertex to the Vertex that must be on the same Chip :type same_chip_vertex_groups: dict(MachineVertex, collection(MachineVertex)) :param ~spinn_machine.Machine machine: The machine with respect to which to partition the application graph :param int plan_n_timesteps: number of timesteps to plan for :param MachineGraph machine_graph: The machine_graph to place :param ~spinn_utilities.progress_bar.ProgressBar progress: :rtype: Placements """ placements = Placements() resource_tracker = ResourceTracker( machine, plan_n_timesteps, self._generate_radial_chips(machine)) all_vertices_placed = set() # RadialPlacementFromChipConstraint won't work here for vertex in machine_graph.vertices: for constraint in vertex.constraints: if isinstance(constraint, RadialPlacementFromChipConstraint): raise PacmanPlaceException( "A RadialPlacementFromChipConstraint will not work " "with the OneToOnePlacer algorithm; use the " "RadialPlacer algorithm instead") # Find and place vertices with hard constraints for vertex in machine_graph.vertices: if isinstance(vertex, AbstractVirtual): virtual_p = 0 while placements.is_processor_occupied( vertex.virtual_chip_x, vertex.virtual_chip_y, virtual_p): virtual_p += 1 placements.add_placement(Placement( vertex, vertex.virtual_chip_x, vertex.virtual_chip_y, virtual_p)) all_vertices_placed.add(vertex) elif locate_constraints_of_type( vertex.constraints, ChipAndCoreConstraint): self._allocate_same_chip_as_group( vertex, placements, resource_tracker, same_chip_vertex_groups, all_vertices_placed, progress, machine_graph) for grouped_vertices in one_to_one_groups: # Get unallocated vertices and placements of allocated vertices unallocated = list() chips = list() for vert in grouped_vertices: if vert in all_vertices_placed: placement = placements.get_placement_of_vertex(vert) chips.append((placement.x, placement.y)) else: unallocated.append(vert) if not chips: chips = None if 0 < len(unallocated) <=\ resource_tracker.get_maximum_cores_available_on_a_chip(): # Try to allocate all vertices to the same chip self._allocate_one_to_one_group( resource_tracker, unallocated, progress, placements, chips, all_vertices_placed, machine_graph) # if too big or failed go on to other groups first # check all have been allocated if not do so now. for vertex in machine_graph.vertices: if vertex not in all_vertices_placed: self._allocate_same_chip_as_group( vertex, placements, resource_tracker, same_chip_vertex_groups, all_vertices_placed, progress, machine_graph) progress.end() return placements
def __call__(self, graph, machine): utility_calls.check_algorithm_can_support_constraints( constrained_vertices=graph.vertices, supported_constraints=[PartitionerMaximumSizeConstraint], abstract_constraint_type=AbstractPartitionerConstraint) # start progress bar progress_bar = ProgressBar(len(graph.vertices), "Partitioning graph vertices") vertices = graph.vertices subgraph = PartitionedGraph(label="partitioned_graph for partitionable" "_graph {}".format(graph.label)) graph_to_subgraph_mapper = GraphMapper(graph.label, subgraph.label) resource_tracker = ResourceTracker(machine) # Partition one vertex at a time for vertex in vertices: # Get the usage of the first atom, then assume that this # will be the usage of all the atoms requirements = vertex.get_resources_used_by_atoms(Slice(0, 1), graph) # Locate the maximum resources available max_resources_available = \ resource_tracker.get_maximum_constrained_resources_available( vertex.constraints) # Find the ratio of each of the resources - if 0 is required, # assume the ratio is the max available atoms_per_sdram = self._get_ratio( max_resources_available.sdram.get_value(), requirements.sdram.get_value()) atoms_per_dtcm = self._get_ratio( max_resources_available.dtcm.get_value(), requirements.dtcm.get_value()) atoms_per_cpu = self._get_ratio( max_resources_available.cpu.get_value(), requirements.cpu.get_value()) max_atom_values = [atoms_per_sdram, atoms_per_dtcm, atoms_per_cpu] max_atoms_constraints = utility_calls.locate_constraints_of_type( vertex.constraints, PartitionerMaximumSizeConstraint) for max_atom_constraint in max_atoms_constraints: max_atom_values.append(max_atom_constraint.size) atoms_per_core = min(max_atom_values) # Partition into subvertices counted = 0 while counted < vertex.n_atoms: # Determine subvertex size remaining = vertex.n_atoms - counted if remaining > atoms_per_core: alloc = atoms_per_core else: alloc = remaining # Create and store new subvertex, and increment elements # counted if counted < 0 or counted + alloc - 1 < 0: raise PacmanPartitionException("Not enough resources" " available to create" " subvertex") vertex_slice = Slice(counted, counted + (alloc - 1)) subvertex_usage = vertex.get_resources_used_by_atoms( vertex_slice, graph) subvert = vertex.create_subvertex( vertex_slice, subvertex_usage, "{}:{}:{}".format(vertex.label, counted, (counted + (alloc - 1))), partition_algorithm_utilities. get_remaining_constraints(vertex)) subgraph.add_subvertex(subvert) graph_to_subgraph_mapper.add_subvertex( subvert, vertex_slice, vertex) counted = counted + alloc # update allocated resources resource_tracker.allocate_constrained_resources( subvertex_usage, vertex.constraints) # update and end progress bars as needed progress_bar.update() progress_bar.end() partition_algorithm_utilities.generate_sub_edges( subgraph, graph_to_subgraph_mapper, graph) return {'Partitioned_graph': subgraph, 'Graph_mapper': graph_to_subgraph_mapper}
def __call__(self, machine_graph, n_keys_map, routing_tables): # check that this algorithm supports the constraints check_algorithm_can_support_constraints( constrained_vertices=machine_graph.outgoing_edge_partitions, supported_constraints=[ FixedMaskConstraint, FixedKeyAndMaskConstraint, ContiguousKeyRangeContraint], abstract_constraint_type=AbstractKeyAllocatorConstraint) # verify that no edge has more than 1 of a constraint ,and that # constraints are compatible check_types_of_edge_constraint(machine_graph) routing_infos = RoutingInfo() # Get the edges grouped by those that require the same key (fixed_keys, _shared_keys, fixed_masks, fixed_fields, flexi_fields, continuous, noncontinuous) = \ get_edge_groups(machine_graph, EdgeTrafficType.MULTICAST) if flexi_fields: raise PacmanConfigurationException( "MallocBasedRoutingInfoAllocator does not support FlexiField") # Even non-continuous keys will be continuous for group in noncontinuous: continuous.add(group) # Go through the groups and allocate keys progress = ProgressBar( machine_graph.n_outgoing_edge_partitions, "Allocating routing keys") # allocate the groups that have fixed keys for group in progress.over(fixed_keys, False): # Get any fixed keys and masks from the group and attempt to # allocate them fixed_mask = None fixed_key_and_mask_constraint = locate_constraints_of_type( group.constraints, FixedKeyAndMaskConstraint)[0] # attempt to allocate them self._allocate_fixed_keys_and_masks( fixed_key_and_mask_constraint.keys_and_masks, fixed_mask) # update the pacman data objects self._update_routing_objects( fixed_key_and_mask_constraint.keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_masks, False): # get mask and fields if need be fixed_mask = locate_constraints_of_type( group.constraints, FixedMaskConstraint)[0].mask fields = None if group in fixed_fields: fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields fixed_fields.remove(group) # try to allocate keys_and_masks = self._allocate_keys_and_masks( fixed_mask, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_fields, False): fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields # try to allocate keys_and_masks = self._allocate_keys_and_masks( None, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) # Sort the rest of the groups, using the routing tables for guidance # Group partitions by those which share routes in any table partition_groups = OrderedDict() routers = reversed(sorted( routing_tables.get_routers(), key=lambda item: len(routing_tables.get_entries_for_router( item[0], item[1])))) for x, y in routers: # Find all partitions that share a route in this table partitions_by_route = defaultdict(OrderedSet) routing_table = routing_tables.get_entries_for_router(x, y) for partition, entry in iteritems(routing_table): if partition in continuous: entry_hash = sum( 1 << i for i in entry.link_ids) entry_hash += sum( 1 << (i + 6) for i in entry.processor_ids) partitions_by_route[entry_hash].add(partition) for entry_hash, partitions in iteritems(partitions_by_route): found_groups = list() for partition in partitions: if partition in partition_groups: found_groups.append(partition_groups[partition]) if not found_groups: # If no group was found, create a new one for partition in partitions: partition_groups[partition] = partitions elif len(found_groups) == 1: # If a single other group was found, merge it for partition in partitions: found_groups[0].add(partition) partition_groups[partition] = found_groups[0] else: # Merge the groups new_group = partitions for group in found_groups: for partition in group: new_group.add(partition) for partition in new_group: partition_groups[partition] = new_group # Sort partitions by largest group continuous = list(OrderedSet( tuple(group) for group in itervalues(partition_groups))) for group in reversed(sorted(continuous, key=len)): for partition in progress.over(group, False): keys_and_masks = self._allocate_keys_and_masks( None, None, n_keys_map.n_keys_for_partition(partition)) # update the pacman data objects self._update_routing_objects( keys_and_masks, routing_infos, partition) progress.end() return routing_infos
def __call__(self, machine, placements): """ see AbstractTagAllocatorAlgorithm.allocate_tags """ resource_tracker = ResourceTracker(machine) # Check that the algorithm can handle the constraints progress_bar = ProgressBar(placements.n_placements, "Allocating tags") placements_with_tags = list() for placement in placements.placements: utility_calls.check_algorithm_can_support_constraints( constrained_vertices=[placement.subvertex], supported_constraints=[ TagAllocatorRequireIptagConstraint, TagAllocatorRequireReverseIptagConstraint ], abstract_constraint_type=AbstractTagAllocatorConstraint) if len(utility_calls.locate_constraints_of_type( placement.subvertex.constraints, AbstractTagAllocatorConstraint)): placements_with_tags.append(placement) progress_bar.update() # Go through and allocate the tags tags = Tags() for placement in placements_with_tags: vertex = placement.subvertex # Get the constraint details for the tags (board_address, ip_tags, reverse_ip_tags) =\ utility_calls.get_ip_tag_info(vertex.constraints) # Allocate the tags, first-come, first-served, using the # fixed placement of the vertex, and the required resources chips = [(placement.x, placement.y)] resources = vertex.resources_required (_, _, _, returned_ip_tags, returned_reverse_ip_tags) = \ resource_tracker.allocate_resources( resources, chips, placement.p, board_address, ip_tags, reverse_ip_tags) # Put the allocated ip tag information into the tag object if returned_ip_tags is not None: for (tag_constraint, (board_address, tag)) in zip( ip_tags, returned_ip_tags): ip_tag = IPTag( board_address, tag, tag_constraint.ip_address, tag_constraint.port, tag_constraint.strip_sdp) tags.add_ip_tag(ip_tag, vertex) # Put the allocated reverse ip tag information into the tag object if returned_reverse_ip_tags is not None: for (tag_constraint, (board_address, tag)) in zip( reverse_ip_tags, returned_reverse_ip_tags): reverse_ip_tag = ReverseIPTag( board_address, tag, tag_constraint.port, placement.x, placement.y, placement.p, tag_constraint.sdp_port) tags.add_reverse_ip_tag(reverse_ip_tag, vertex) progress_bar.end() return {'tags': tags}
def _partition_vertex( self, vertex, plan_n_timesteps, machine_graph, graph_mapper, resource_tracker, progress, vertex_groups): """ Partition a single vertex :param vertex: the vertex to partition :type vertex:\ :py:class:`pacman.model.graphs.application.ApplicationVertex` :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :param machine_graph: the graph to add vertices to :type machine_graph:\ :py:class:`pacman.model.graphs.machine.MachineGraph` :param graph_mapper: the mappings between graphs :type graph_mapper:\ :py:class:`pacman.model.graphs.common.GraphMapper' :param resource_tracker: A tracker of assigned resources :type resource_tracker:\ :py:class:`pacman.utilities.ResourceTracker` :param progress: The progress bar :param vertex_groups: Groups together vertices that are supposed to\ be the same size :rtype: None :raise pacman.exceptions.PacmanPartitionException: \ if the extra vertex for partitioning identically has a different\ number of atoms than its counterpart. """ partition_together_vertices = list(vertex_groups[vertex]) # locate max atoms per core and fixed atoms per core possible_max_atoms = list() n_atoms = None for other_vertex in partition_together_vertices: if isinstance(other_vertex, ApplicationVertex): possible_max_atoms.append( other_vertex.get_max_atoms_per_core()) max_atom_constraints = utils.locate_constraints_of_type( other_vertex.constraints, MaxVertexAtomsConstraint) for constraint in max_atom_constraints: possible_max_atoms.append(constraint.size) n_atom_constraints = utils.locate_constraints_of_type( other_vertex.constraints, FixedVertexAtomsConstraint) for constraint in n_atom_constraints: if n_atoms is not None and constraint.size != n_atoms: raise PacmanPartitionException( "Vertex has multiple contradictory fixed atom " "constraints - cannot be both {} and {}".format( n_atoms, constraint.size)) n_atoms = constraint.size max_atoms_per_core = int(min(possible_max_atoms)) if n_atoms is not None and max_atoms_per_core < n_atoms: raise PacmanPartitionException( "Max size of {} is incompatible with fixed size of {}".format( max_atoms_per_core, n_atoms)) if n_atoms is not None: max_atoms_per_core = n_atoms if vertex.n_atoms % n_atoms != 0: raise PacmanPartitionException( "Vertex of {} atoms cannot be divided into units of {}" .format(vertex.n_atoms, n_atoms)) # partition by atoms self._partition_by_atoms( partition_together_vertices, plan_n_timesteps, vertex.n_atoms, max_atoms_per_core, machine_graph, graph_mapper, resource_tracker, progress, n_atoms is not None)
def __call__(self, partitioned_graph, file_path): """ :param partitioned_graph: :param folder_path: :return: """ progress_bar = ProgressBar(len(partitioned_graph.subvertices), "Converting to json partitioned graph") # write basic stuff json_graph_dictory_rep = dict() # write vertices data vertices_resources = dict() json_graph_dictory_rep["vertices_resources"] = vertices_resources edges_resources = defaultdict() json_graph_dictory_rep["edges"] = edges_resources for vertex in partitioned_graph.subvertices: # handle external devices if isinstance(vertex, AbstractVirtualVertex): vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = 0 # handle tagged vertices elif len( utility_calls.locate_constraints_of_type( vertex.constraints, AbstractTagAllocatorConstraint)) != 0: # handle the edge between the tag-able vertex and the fake # vertex hyper_edge_dict = dict() edges_resources[hashlib.md5(vertex.label).hexdigest()] = \ hyper_edge_dict hyper_edge_dict["source"] = str(id(vertex)) hyper_edge_dict['sinks'] = \ [hashlib.md5(vertex.label).hexdigest()] hyper_edge_dict["weight"] = 1.0 hyper_edge_dict["type"] = "FAKE_TAG_EDGE" # add the tag-able vertex vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = \ DEFAULT_NOUMBER_OF_CORES_USED_PER_PARTITIONED_VERTEX vertex_resources["sdram"] = \ int(vertex.resources_required.sdram.get_value()) # add fake vertex vertex_resources = dict() vertices_resources[hashlib.md5( vertex.label).hexdigest()] = vertex_resources vertex_resources["cores"] = 0 vertex_resources["sdram"] = 0 # handle standard vertices else: vertex_resources = dict() vertices_resources[id(vertex)] = vertex_resources vertex_resources["cores"] = \ DEFAULT_NOUMBER_OF_CORES_USED_PER_PARTITIONED_VERTEX vertex_resources["sdram"] = \ int(vertex.resources_required.sdram.get_value()) vertex_outgoing_partitions = \ partitioned_graph.outgoing_edges_partitions_from_vertex(vertex) # handle the vertex edges for partition_id in vertex_outgoing_partitions: partition = vertex_outgoing_partitions[partition_id] hyper_edge_dict = dict() edges_resources[str(id(partition))] = hyper_edge_dict hyper_edge_dict["source"] = str(id(vertex)) sinks_string = [] weight = 0 for edge in partition.edges: sinks_string.append(str(id(edge.post_subvertex))) weight += edge.weight hyper_edge_dict['sinks'] = sinks_string hyper_edge_dict["weight"] = weight hyper_edge_dict["type"] = partition.type.name.lower() progress_bar.update() file_to_write = open(file_path, "w") json.dump(json_graph_dictory_rep, file_to_write) file_to_write.close() # validate the schema partitioned_graph_schema_file_path = os.path.join( os.path.dirname(file_format_schemas.__file__), "partitioned_graph.json") file_to_read = open(partitioned_graph_schema_file_path, "r") partitioned_graph_schema = json.load(file_to_read) jsonschema.validate(json_graph_dictory_rep, partitioned_graph_schema) progress_bar.end() return {"partitioned_graph": file_path}
def check_types_of_edge_constraint(machine_graph): """ Go through the graph for operations and checks that the constraints\ are compatible. :param machine_graph: the graph to search through :rtype: None: """ for partition in machine_graph.outgoing_edge_partitions: fixed_key = locate_constraints_of_type( partition.constraints, FixedKeyAndMaskConstraint) fixed_mask = locate_constraints_of_type( partition.constraints, FixedMaskConstraint) fixed_field = locate_constraints_of_type( partition.constraints, FixedKeyFieldConstraint) flexi_field = locate_constraints_of_type( partition.constraints, FlexiKeyFieldConstraint) if (len(fixed_key) > 1 or len(fixed_field) > 1 or len(fixed_mask) > 1 or len(flexi_field) > 1): raise PacmanConfigurationException( "There are more than one of the same constraint type on " "the partition {} starting at {}. Please fix and try again." .format(partition.identifier, partition.pre_vertex)) fixed_key = len(fixed_key) == 1 fixed_mask = len(fixed_mask) == 1 fixed_field = len(fixed_field) == 1 flexi_field = len(flexi_field) == 1 # check for fixed key and a fixed mask. as these should have been # merged before now if fixed_key and fixed_mask: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "mask constraint. These can be merged together, but is " "deemed an error here" .format(partition.identifer, partition.pre_vertex)) # check for a fixed key and fixed field, as these are incompatible if fixed_key and fixed_field: raise PacmanConfigurationException( "The partition {} starting at {} has a fixed key and fixed " "field constraint. These may be merge-able together, but " "is deemed an error here" .format(partition.identifer, partition.pre_vertex)) # check that a fixed mask and fixed field have compatible masks if fixed_mask and fixed_field: _check_masks_are_correct(partition) # check that if there's a flexible field, and something else, throw # error if flexi_field and (fixed_mask or fixed_key or fixed_field): raise PacmanConfigurationException( "The partition {} starting at {} has a flexible field and " "another fixed constraint. These maybe be merge-able, but " "is deemed an error here" .format(partition.identifer, partition.pre_vertex))
def __call__(self, machine_graph, n_keys_map, routing_tables): """ :param MachineGraph machine_graph: :param AbstractMachinePartitionNKeysMap n_keys_map: :param MulticastRoutingTableByPartition routing_tables: :rtype: RoutingInfo """ # check that this algorithm supports the constraints check_algorithm_can_support_constraints( constrained_vertices=machine_graph.outgoing_edge_partitions, supported_constraints=[ FixedMaskConstraint, FixedKeyAndMaskConstraint, ContiguousKeyRangeContraint ], abstract_constraint_type=AbstractKeyAllocatorConstraint) # verify that no edge has more than 1 of a constraint ,and that # constraints are compatible check_types_of_edge_constraint(machine_graph) routing_infos = RoutingInfo() # Get the edges grouped by those that require the same key (fixed_keys, _shared_keys, fixed_masks, fixed_fields, continuous, noncontinuous) = get_mulitcast_edge_groups(machine_graph) # Even non-continuous keys will be continuous continuous.extend(noncontinuous) # Go through the groups and allocate keys progress = ProgressBar(machine_graph.n_outgoing_edge_partitions, "Allocating routing keys") # allocate the groups that have fixed keys for group in progress.over(fixed_keys, False): # Get any fixed keys and masks from the group and attempt to # allocate them fixed_mask = None fixed_key_and_mask_constraint = locate_constraints_of_type( group.constraints, FixedKeyAndMaskConstraint)[0] # attempt to allocate them self._allocate_fixed_keys_and_masks( fixed_key_and_mask_constraint.keys_and_masks, fixed_mask) # update the pacman data objects self._update_routing_objects( fixed_key_and_mask_constraint.keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_masks, False): # get mask and fields if need be fixed_mask = locate_constraints_of_type( group.constraints, FixedMaskConstraint)[0].mask fields = None if group in fixed_fields: fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields fixed_fields.remove(group) # try to allocate keys_and_masks = self._allocate_keys_and_masks( fixed_mask, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_fields, False): fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields # try to allocate keys_and_masks = self._allocate_keys_and_masks( None, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) # Sort the rest of the groups, using the routing tables for guidance # Group partitions by those which share routes in any table partition_groups = OrderedDict() routers = reversed( sorted( routing_tables.get_routers(), key=lambda item: len( routing_tables.get_entries_for_router(item[0], item[1])))) for x, y in routers: # Find all partitions that share a route in this table partitions_by_route = defaultdict(OrderedSet) routing_table = routing_tables.get_entries_for_router(x, y) for partition, entry in iteritems(routing_table): if partition in continuous: entry_hash = sum(1 << i for i in entry.link_ids) entry_hash += sum(1 << (i + 6) for i in entry.processor_ids) partitions_by_route[entry_hash].add(partition) for entry_hash, partitions in iteritems(partitions_by_route): found_groups = list() for partition in partitions: if partition in partition_groups: found_groups.append(partition_groups[partition]) if not found_groups: # If no group was found, create a new one for partition in partitions: partition_groups[partition] = partitions elif len(found_groups) == 1: # If a single other group was found, merge it for partition in partitions: found_groups[0].add(partition) partition_groups[partition] = found_groups[0] else: # Merge the groups new_group = partitions for group in found_groups: for partition in group: new_group.add(partition) for partition in new_group: partition_groups[partition] = new_group # Sort partitions by largest group continuous = list( OrderedSet(tuple(group) for group in itervalues(partition_groups))) for group in reversed(sorted(continuous, key=len)): for partition in progress.over(group, False): keys_and_masks = self._allocate_keys_and_masks( None, None, n_keys_map.n_keys_for_partition(partition)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, partition) progress.end() return routing_infos