def __init__(self,
region,
sent_stop_message=False,
n_sequences_per_tranmission=64):
"""
:param region: The region being managed
:type region: int
:param sent_stop_message: True if the stop message has been sent
:type sent_stop_message: bool
:param n_sequences_per_tranmission: \
The number of sequences allowed in each transmission set
:type n_sequences_per_tranmission: int
"""
self._region = region
# A queue of messages sent, ordered by sequence number
self._buffers_sent = deque(maxlen=n_sequences_per_tranmission)
# The current sequence number of the region
self._sequence_number = 0
# A lock for the sequence number
self._sequence_lock = Lock()
# The last sequence number to be received on the machine
self._last_received_sequence_number = _N_SEQUENCES - 1
# True if the stop message has been sent
self._sent_stop_message = sent_stop_message
# The number of sequence numbers allowed in a single transmission
self._n_sequences_per_transmission = n_sequences_per_tranmission
def __init__(self, hostname, port=22244, timeout=None):
""" Define a new connection.
.. note::
Does not connect to the server until :py:meth:`.connect` is called.
Parameters
----------
hostname : str
The hostname of the server.
port : str
The port to use (default: 22244).
"""
self._hostname = hostname
self._port = port
# Mapping from threads to sockets. Kept because we need to have way to
# shut down all sockets at once.
self._socks = dict()
# Thread local variables
self._local = _ProtocolThreadLocal()
# A queue of unprocessed notifications
self._notifications = deque()
self._dead = True
self._socks_lock = RLock()
self._notifications_lock = RLock()
self._default_timeout = timeout
def __init__(self, region, sent_stop_message=False,
n_sequences_per_tranmission=64):
"""
:param region: The region being managed
:type region: int
:param sent_stop_message: True if the stop message has been sent
:type sent_stop_message: bool
:param n_sequences_per_tranmission: \
The number of sequences allowed in each transmission set
:type n_sequences_per_tranmission: int
"""
self._region = region
# A queue of messages sent, ordered by sequence number
self._buffers_sent = deque(maxlen=n_sequences_per_tranmission)
# The current sequence number of the region
self._sequence_number = 0
# A lock for the sequence number
self._sequence_lock = Lock()
# The last sequence number to be received on the machine
self._last_received_sequence_number = _N_SEQUENCES - 1
# True if the stop message has been sent
self._sent_stop_message = sent_stop_message
# The number of sequence numbers allowed in a single transmission
self._n_sequences_per_transmission = n_sequences_per_tranmission
def test_routing(self):
graph = MachineGraph("Test")
machine = VirtualMachine(2, 2)
placements = Placements()
vertices = list()
for x in range(machine.max_chip_x + 1):
for y in range(machine.max_chip_y + 1):
chip = machine.get_chip_at(x, y)
if chip is not None:
for processor in chip.processors:
if not processor.is_monitor:
vertex = SimpleMachineVertex(
resources=ResourceContainer())
graph.add_vertex(vertex)
placements.add_placement(Placement(
vertex, x, y, processor.processor_id))
vertices.append(vertex)
for vertex in vertices:
for vertex_to in vertices:
if vertex != vertex_to:
graph.add_edge(MachineEdge(vertex, vertex_to), "Test")
router = BasicDijkstraRouting()
routing_paths = router.__call__(placements, machine, graph)
for vertex in vertices:
vertices_reached = set()
queue = deque()
seen_entries = set()
placement = placements.get_placement_of_vertex(vertex)
partition = graph.get_outgoing_edge_partition_starting_at_vertex(
vertex, "Test")
entry = routing_paths.get_entry_on_coords_for_edge(
partition, placement.x, placement.y)
self.assertEqual(entry.incoming_processor, placement.p)
queue.append((placement.x, placement.y))
while len(queue) > 0:
x, y = queue.pop()
entry = routing_paths.get_entry_on_coords_for_edge(
partition, x, y)
self.assertIsNotNone(entry)
chip = machine.get_chip_at(x, y)
for p in entry.processor_ids:
self.assertIsNotNone(chip.get_processor_with_id(p))
vertex_found = placements.get_vertex_on_processor(x, y, p)
vertices_reached.add(vertex_found)
seen_entries.add((x, y))
for link_id in entry.link_ids:
link = chip.router.get_link(link_id)
self.assertIsNotNone(link)
dest_x, dest_y = link.destination_x, link.destination_y
if (dest_x, dest_y) not in seen_entries:
queue.append((dest_x, dest_y))
for vertex_to in vertices:
if vertex != vertex_to:
self.assertIn(vertex_to, vertices_reached)
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockMachine, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection()
self._running = False
self._messages = deque()
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockMachine, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection()
self._running = False
self._messages = deque()
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
def __init__(self):
MessageHandler.__init__(self)
self.status = None
self.args = None
self._bytes_parts = deque()
self._body_started = False
self._body_stream_status = None
self._body = None
self._body_error_args = None
self.finished_reading = False
def _generate_radial_chips(
machine, resource_tracker=None, start_chip_x=None,
start_chip_y=None):
""" Generates the list of chips from a given starting point in a radial\
format.
:param machine: the SpiNNaker machine object
:param resource_tracker:\
the resource tracker object which contains what resources of the\
machine have currently been used
:type resource_tracker: None or \
:py:class:`ResourceTracker`
:param start_chip_x:\
The chip x coordinate to start with for radial iteration
:param start_chip_y:\
the chip y coordinate to start with for radial iteration
:return: list of chips.
"""
if start_chip_x is None or start_chip_y is None:
first_chip = machine.boot_chip
else:
first_chip = machine.get_chip_at(start_chip_x, start_chip_y)
done_chips = {first_chip}
search = deque([first_chip])
while search:
chip = search.pop()
if (resource_tracker is None or
resource_tracker.is_chip_available(chip.x, chip.y)):
yield chip.x, chip.y
# Examine the links of the chip to find the next chips
for link in chip.router.links:
next_chip = machine.get_chip_at(
link.destination_x, link.destination_y)
# Don't search done chips again
if next_chip not in done_chips:
search.appendleft(next_chip)
done_chips.add(next_chip)
def add_node(self, key, parent_keys):
"""Add a new node to the graph.
If this fills in a ghost, then the gdfos of all children will be
updated accordingly.
:param key: The node being added. If this is a duplicate, this is a
no-op.
:param parent_keys: The parents of the given node.
:return: None (should we return if this was a ghost, etc?)
"""
nodes = self._nodes
if key in nodes:
node = nodes[key]
if node.parent_keys is None:
node.parent_keys = parent_keys
# A ghost is being added, we can no-longer trust the heads
# cache, so clear it
self._known_heads.clear()
else:
# Make sure we compare a list to a list, as tuple != list.
parent_keys = list(parent_keys)
existing_parent_keys = list(node.parent_keys)
if parent_keys == existing_parent_keys:
return # Identical content
else:
raise ValueError('Parent key mismatch, existing node %s'
' has parents of %s not %s' %
(key, existing_parent_keys, parent_keys))
else:
node = _KnownGraphNode(key, parent_keys)
nodes[key] = node
parent_gdfo = 0
for parent_key in parent_keys:
try:
parent_node = nodes[parent_key]
except KeyError:
parent_node = _KnownGraphNode(parent_key, None)
# Ghosts and roots have gdfo 1
parent_node.gdfo = 1
nodes[parent_key] = parent_node
if parent_gdfo < parent_node.gdfo:
parent_gdfo = parent_node.gdfo
parent_node.child_keys.append(key)
node.gdfo = parent_gdfo + 1
# Now fill the gdfo to all children
# Note that this loop is slightly inefficient, in that we may visit the
# same child (and its decendents) more than once, however, it is
# 'efficient' in that we only walk to nodes that would be updated,
# rather than all nodes
# We use a deque rather than a simple list stack, to go for BFD rather
# than DFD. So that if a longer path is possible, we walk it before we
# get to the final child
pending = deque([node])
while pending:
node = pending.popleft()
next_gdfo = node.gdfo + 1
for child_key in node.child_keys:
child = nodes[child_key]
if child.gdfo < next_gdfo:
# This child is being updated, we need to check its
# children
child.gdfo = next_gdfo
pending.append(child)
def iter_changes(self, include_unchanged=False,
specific_files=None, pb=None, extra_trees=[],
require_versioned=True, want_unversioned=False):
"""Generate an iterator of changes between trees.
A tuple is returned:
(file_id, (path_in_source, path_in_target),
changed_content, versioned, parent, name, kind,
executable)
Changed_content is True if the file's content has changed. This
includes changes to its kind, and to a symlink's target.
versioned, parent, name, kind, executable are tuples of (from, to).
If a file is missing in a tree, its kind is None.
Iteration is done in parent-to-child order, relative to the target
tree.
There is no guarantee that all paths are in sorted order: the
requirement to expand the search due to renames may result in children
that should be found early being found late in the search, after
lexically later results have been returned.
:param require_versioned: Raise errors.PathsNotVersionedError if a
path in the specific_files list is not versioned in one of
source, target or extra_trees.
:param specific_files: An optional list of file paths to restrict the
comparison to. When mapping filenames to ids, all matches in all
trees (including optional extra_trees) are used, and all children
of matched directories are included. The parents in the target tree
of the specific files up to and including the root of the tree are
always evaluated for changes too.
:param want_unversioned: Should unversioned files be returned in the
output. An unversioned file is defined as one with (False, False)
for the versioned pair.
"""
if not extra_trees:
extra_trees = []
else:
extra_trees = list(extra_trees)
# The ids of items we need to examine to insure delta consistency.
precise_file_ids = set()
changed_file_ids = []
if specific_files == []:
target_specific_files = []
source_specific_files = []
else:
target_specific_files = self.target.find_related_paths_across_trees(
specific_files, [self.source] + extra_trees,
require_versioned=require_versioned)
source_specific_files = self.source.find_related_paths_across_trees(
specific_files, [self.target] + extra_trees,
require_versioned=require_versioned)
if specific_files is not None:
# reparented or added entries must have their parents included
# so that valid deltas can be created. The seen_parents set
# tracks the parents that we need to have.
# The seen_dirs set tracks directory entries we've yielded.
# After outputting version object in to_entries we set difference
# the two seen sets and start checking parents.
seen_parents = set()
seen_dirs = set()
if want_unversioned:
all_unversioned = sorted([(p.split('/'), p) for p in
self.target.extras()
if specific_files is None or
osutils.is_inside_any(specific_files, p)])
all_unversioned = deque(all_unversioned)
else:
all_unversioned = deque()
to_paths = {}
from_entries_by_dir = list(self.source.iter_entries_by_dir(
specific_files=source_specific_files))
from_data = dict(from_entries_by_dir)
to_entries_by_dir = list(self.target.iter_entries_by_dir(
specific_files=target_specific_files))
path_equivs = find_previous_paths(
self.target, self.source, [p for p, e in to_entries_by_dir])
num_entries = len(from_entries_by_dir) + len(to_entries_by_dir)
entry_count = 0
# the unversioned path lookup only occurs on real trees - where there
# can be extras. So the fake_entry is solely used to look up
# executable it values when execute is not supported.
fake_entry = TreeFile()
for target_path, target_entry in to_entries_by_dir:
while (all_unversioned and
all_unversioned[0][0] < target_path.split('/')):
unversioned_path = all_unversioned.popleft()
target_kind, target_executable, target_stat = \
self.target._comparison_data(
fake_entry, unversioned_path[1])
yield TreeChange(
None, (None, unversioned_path[1]), True, (False, False),
(None, None),
(None, unversioned_path[0][-1]),
(None, target_kind),
(None, target_executable))
source_path = path_equivs[target_path]
if source_path is not None:
source_entry = from_data.get(source_path)
else:
source_entry = None
result, changes = self._changes_from_entries(
source_entry, target_entry, source_path=source_path, target_path=target_path)
to_paths[result.file_id] = result.path[1]
entry_count += 1
if result.versioned[0]:
entry_count += 1
if pb is not None:
pb.update('comparing files', entry_count, num_entries)
if changes or include_unchanged:
if specific_files is not None:
precise_file_ids.add(result.parent_id[1])
changed_file_ids.append(result.file_id)
yield result
# Ensure correct behaviour for reparented/added specific files.
if specific_files is not None:
# Record output dirs
if result.kind[1] == 'directory':
seen_dirs.add(result.file_id)
# Record parents of reparented/added entries.
if not result.versioned[0] or result.is_reparented():
seen_parents.add(result.parent_id[1])
while all_unversioned:
# yield any trailing unversioned paths
unversioned_path = all_unversioned.popleft()
to_kind, to_executable, to_stat = \
self.target._comparison_data(fake_entry, unversioned_path[1])
yield TreeChange(
None, (None, unversioned_path[1]), True, (False, False),
(None, None),
(None, unversioned_path[0][-1]),
(None, to_kind),
(None, to_executable))
# Yield all remaining source paths
for path, from_entry in from_entries_by_dir:
file_id = from_entry.file_id
if file_id in to_paths:
# already returned
continue
to_path = find_previous_path(self.source, self.target, path)
entry_count += 1
if pb is not None:
pb.update('comparing files', entry_count, num_entries)
versioned = (True, False)
parent = (from_entry.parent_id, None)
name = (from_entry.name, None)
from_kind, from_executable, stat_value = \
self.source._comparison_data(from_entry, path)
kind = (from_kind, None)
executable = (from_executable, None)
changed_content = from_kind is not None
# the parent's path is necessarily known at this point.
changed_file_ids.append(file_id)
yield TreeChange(
file_id, (path, to_path), changed_content, versioned, parent,
name, kind, executable)
changed_file_ids = set(changed_file_ids)
if specific_files is not None:
for result in self._handle_precise_ids(precise_file_ids,
changed_file_ids):
yield result