def __init__(self, region, sent_stop_message=False, n_sequences_per_tranmission=64): """ :param region: The region being managed :type region: int :param sent_stop_message: True if the stop message has been sent :type sent_stop_message: bool :param n_sequences_per_tranmission: \ The number of sequences allowed in each transmission set :type n_sequences_per_tranmission: int """ self._region = region # A queue of messages sent, ordered by sequence number self._buffers_sent = deque(maxlen=n_sequences_per_tranmission) # The current sequence number of the region self._sequence_number = 0 # A lock for the sequence number self._sequence_lock = Lock() # The last sequence number to be received on the machine self._last_received_sequence_number = _N_SEQUENCES - 1 # True if the stop message has been sent self._sent_stop_message = sent_stop_message # The number of sequence numbers allowed in a single transmission self._n_sequences_per_transmission = n_sequences_per_tranmission
def __init__(self, hostname, port=22244, timeout=None): """ Define a new connection. .. note:: Does not connect to the server until :py:meth:`.connect` is called. Parameters ---------- hostname : str The hostname of the server. port : str The port to use (default: 22244). """ self._hostname = hostname self._port = port # Mapping from threads to sockets. Kept because we need to have way to # shut down all sockets at once. self._socks = dict() # Thread local variables self._local = _ProtocolThreadLocal() # A queue of unprocessed notifications self._notifications = deque() self._dead = True self._socks_lock = RLock() self._notifications_lock = RLock() self._default_timeout = timeout
def test_routing(self): graph = MachineGraph("Test") machine = VirtualMachine(2, 2) placements = Placements() vertices = list() for x in range(machine.max_chip_x + 1): for y in range(machine.max_chip_y + 1): chip = machine.get_chip_at(x, y) if chip is not None: for processor in chip.processors: if not processor.is_monitor: vertex = SimpleMachineVertex( resources=ResourceContainer()) graph.add_vertex(vertex) placements.add_placement(Placement( vertex, x, y, processor.processor_id)) vertices.append(vertex) for vertex in vertices: for vertex_to in vertices: if vertex != vertex_to: graph.add_edge(MachineEdge(vertex, vertex_to), "Test") router = BasicDijkstraRouting() routing_paths = router.__call__(placements, machine, graph) for vertex in vertices: vertices_reached = set() queue = deque() seen_entries = set() placement = placements.get_placement_of_vertex(vertex) partition = graph.get_outgoing_edge_partition_starting_at_vertex( vertex, "Test") entry = routing_paths.get_entry_on_coords_for_edge( partition, placement.x, placement.y) self.assertEqual(entry.incoming_processor, placement.p) queue.append((placement.x, placement.y)) while len(queue) > 0: x, y = queue.pop() entry = routing_paths.get_entry_on_coords_for_edge( partition, x, y) self.assertIsNotNone(entry) chip = machine.get_chip_at(x, y) for p in entry.processor_ids: self.assertIsNotNone(chip.get_processor_with_id(p)) vertex_found = placements.get_vertex_on_processor(x, y, p) vertices_reached.add(vertex_found) seen_entries.add((x, y)) for link_id in entry.link_ids: link = chip.router.get_link(link_id) self.assertIsNotNone(link) dest_x, dest_y = link.destination_x, link.destination_y if (dest_x, dest_y) not in seen_entries: queue.append((dest_x, dest_y)) for vertex_to in vertices: if vertex != vertex_to: self.assertIn(vertex_to, vertices_reached)
def __init__(self, responses=None): """ :param responses:\ An optional list of responses to send in the order to be sent. \ If not specified, OK responses will be sent for every request. \ Note that responses can include "None" which means that no\ response will be sent to that request """ super(MockMachine, self).__init__() # Set up a connection to be the machine self._receiver = UDPConnection() self._running = False self._messages = deque() self._error = None self._responses = deque() if responses is not None: self._responses.extend(responses)
def __init__(self): MessageHandler.__init__(self) self.status = None self.args = None self._bytes_parts = deque() self._body_started = False self._body_stream_status = None self._body = None self._body_error_args = None self.finished_reading = False
def _generate_radial_chips( machine, resource_tracker=None, start_chip_x=None, start_chip_y=None): """ Generates the list of chips from a given starting point in a radial\ format. :param machine: the SpiNNaker machine object :param resource_tracker:\ the resource tracker object which contains what resources of the\ machine have currently been used :type resource_tracker: None or \ :py:class:`ResourceTracker` :param start_chip_x:\ The chip x coordinate to start with for radial iteration :param start_chip_y:\ the chip y coordinate to start with for radial iteration :return: list of chips. """ if start_chip_x is None or start_chip_y is None: first_chip = machine.boot_chip else: first_chip = machine.get_chip_at(start_chip_x, start_chip_y) done_chips = {first_chip} search = deque([first_chip]) while search: chip = search.pop() if (resource_tracker is None or resource_tracker.is_chip_available(chip.x, chip.y)): yield chip.x, chip.y # Examine the links of the chip to find the next chips for link in chip.router.links: next_chip = machine.get_chip_at( link.destination_x, link.destination_y) # Don't search done chips again if next_chip not in done_chips: search.appendleft(next_chip) done_chips.add(next_chip)
def add_node(self, key, parent_keys): """Add a new node to the graph. If this fills in a ghost, then the gdfos of all children will be updated accordingly. :param key: The node being added. If this is a duplicate, this is a no-op. :param parent_keys: The parents of the given node. :return: None (should we return if this was a ghost, etc?) """ nodes = self._nodes if key in nodes: node = nodes[key] if node.parent_keys is None: node.parent_keys = parent_keys # A ghost is being added, we can no-longer trust the heads # cache, so clear it self._known_heads.clear() else: # Make sure we compare a list to a list, as tuple != list. parent_keys = list(parent_keys) existing_parent_keys = list(node.parent_keys) if parent_keys == existing_parent_keys: return # Identical content else: raise ValueError('Parent key mismatch, existing node %s' ' has parents of %s not %s' % (key, existing_parent_keys, parent_keys)) else: node = _KnownGraphNode(key, parent_keys) nodes[key] = node parent_gdfo = 0 for parent_key in parent_keys: try: parent_node = nodes[parent_key] except KeyError: parent_node = _KnownGraphNode(parent_key, None) # Ghosts and roots have gdfo 1 parent_node.gdfo = 1 nodes[parent_key] = parent_node if parent_gdfo < parent_node.gdfo: parent_gdfo = parent_node.gdfo parent_node.child_keys.append(key) node.gdfo = parent_gdfo + 1 # Now fill the gdfo to all children # Note that this loop is slightly inefficient, in that we may visit the # same child (and its decendents) more than once, however, it is # 'efficient' in that we only walk to nodes that would be updated, # rather than all nodes # We use a deque rather than a simple list stack, to go for BFD rather # than DFD. So that if a longer path is possible, we walk it before we # get to the final child pending = deque([node]) while pending: node = pending.popleft() next_gdfo = node.gdfo + 1 for child_key in node.child_keys: child = nodes[child_key] if child.gdfo < next_gdfo: # This child is being updated, we need to check its # children child.gdfo = next_gdfo pending.append(child)
def iter_changes(self, include_unchanged=False, specific_files=None, pb=None, extra_trees=[], require_versioned=True, want_unversioned=False): """Generate an iterator of changes between trees. A tuple is returned: (file_id, (path_in_source, path_in_target), changed_content, versioned, parent, name, kind, executable) Changed_content is True if the file's content has changed. This includes changes to its kind, and to a symlink's target. versioned, parent, name, kind, executable are tuples of (from, to). If a file is missing in a tree, its kind is None. Iteration is done in parent-to-child order, relative to the target tree. There is no guarantee that all paths are in sorted order: the requirement to expand the search due to renames may result in children that should be found early being found late in the search, after lexically later results have been returned. :param require_versioned: Raise errors.PathsNotVersionedError if a path in the specific_files list is not versioned in one of source, target or extra_trees. :param specific_files: An optional list of file paths to restrict the comparison to. When mapping filenames to ids, all matches in all trees (including optional extra_trees) are used, and all children of matched directories are included. The parents in the target tree of the specific files up to and including the root of the tree are always evaluated for changes too. :param want_unversioned: Should unversioned files be returned in the output. An unversioned file is defined as one with (False, False) for the versioned pair. """ if not extra_trees: extra_trees = [] else: extra_trees = list(extra_trees) # The ids of items we need to examine to insure delta consistency. precise_file_ids = set() changed_file_ids = [] if specific_files == []: target_specific_files = [] source_specific_files = [] else: target_specific_files = self.target.find_related_paths_across_trees( specific_files, [self.source] + extra_trees, require_versioned=require_versioned) source_specific_files = self.source.find_related_paths_across_trees( specific_files, [self.target] + extra_trees, require_versioned=require_versioned) if specific_files is not None: # reparented or added entries must have their parents included # so that valid deltas can be created. The seen_parents set # tracks the parents that we need to have. # The seen_dirs set tracks directory entries we've yielded. # After outputting version object in to_entries we set difference # the two seen sets and start checking parents. seen_parents = set() seen_dirs = set() if want_unversioned: all_unversioned = sorted([(p.split('/'), p) for p in self.target.extras() if specific_files is None or osutils.is_inside_any(specific_files, p)]) all_unversioned = deque(all_unversioned) else: all_unversioned = deque() to_paths = {} from_entries_by_dir = list(self.source.iter_entries_by_dir( specific_files=source_specific_files)) from_data = dict(from_entries_by_dir) to_entries_by_dir = list(self.target.iter_entries_by_dir( specific_files=target_specific_files)) path_equivs = find_previous_paths( self.target, self.source, [p for p, e in to_entries_by_dir]) num_entries = len(from_entries_by_dir) + len(to_entries_by_dir) entry_count = 0 # the unversioned path lookup only occurs on real trees - where there # can be extras. So the fake_entry is solely used to look up # executable it values when execute is not supported. fake_entry = TreeFile() for target_path, target_entry in to_entries_by_dir: while (all_unversioned and all_unversioned[0][0] < target_path.split('/')): unversioned_path = all_unversioned.popleft() target_kind, target_executable, target_stat = \ self.target._comparison_data( fake_entry, unversioned_path[1]) yield TreeChange( None, (None, unversioned_path[1]), True, (False, False), (None, None), (None, unversioned_path[0][-1]), (None, target_kind), (None, target_executable)) source_path = path_equivs[target_path] if source_path is not None: source_entry = from_data.get(source_path) else: source_entry = None result, changes = self._changes_from_entries( source_entry, target_entry, source_path=source_path, target_path=target_path) to_paths[result.file_id] = result.path[1] entry_count += 1 if result.versioned[0]: entry_count += 1 if pb is not None: pb.update('comparing files', entry_count, num_entries) if changes or include_unchanged: if specific_files is not None: precise_file_ids.add(result.parent_id[1]) changed_file_ids.append(result.file_id) yield result # Ensure correct behaviour for reparented/added specific files. if specific_files is not None: # Record output dirs if result.kind[1] == 'directory': seen_dirs.add(result.file_id) # Record parents of reparented/added entries. if not result.versioned[0] or result.is_reparented(): seen_parents.add(result.parent_id[1]) while all_unversioned: # yield any trailing unversioned paths unversioned_path = all_unversioned.popleft() to_kind, to_executable, to_stat = \ self.target._comparison_data(fake_entry, unversioned_path[1]) yield TreeChange( None, (None, unversioned_path[1]), True, (False, False), (None, None), (None, unversioned_path[0][-1]), (None, to_kind), (None, to_executable)) # Yield all remaining source paths for path, from_entry in from_entries_by_dir: file_id = from_entry.file_id if file_id in to_paths: # already returned continue to_path = find_previous_path(self.source, self.target, path) entry_count += 1 if pb is not None: pb.update('comparing files', entry_count, num_entries) versioned = (True, False) parent = (from_entry.parent_id, None) name = (from_entry.name, None) from_kind, from_executable, stat_value = \ self.source._comparison_data(from_entry, path) kind = (from_kind, None) executable = (from_executable, None) changed_content = from_kind is not None # the parent's path is necessarily known at this point. changed_file_ids.append(file_id) yield TreeChange( file_id, (path, to_path), changed_content, versioned, parent, name, kind, executable) changed_file_ids = set(changed_file_ids) if specific_files is not None: for result in self._handle_precise_ids(precise_file_ids, changed_file_ids): yield result