def __init__(self,
start_resume_callback_function=None,
stop_pause_callback_function=None,
local_host=None,
local_port=19999):
"""
:param start_resume_callback_function: A function to be called when
the start message has been received. This function should
not take any parameters or return anything.
:type start_resume_callback_function: function() -> None
:param local_host: Optional specification of the local hostname or\
ip address of the interface to listen on
:type local_host: str
:param local_port: Optional specification of the local port to listen\
on. Must match the port that the toolchain will send the\
notification on (19999 by default)
:type local_port: int
"""
UDPConnection.__init__(self,
local_host=local_host,
local_port=local_port,
remote_host=None,
remote_port=None)
Thread.__init__(self,
name="SpyNNakerDatabaseConnection:{}:{}".format(
self.local_ip_address, self.local_port))
self._database_callback_functions = list()
self._start_resume_callback_function = start_resume_callback_function
self._pause_and_stop_callback_function = stop_pause_callback_function
self._running = False
self.daemon = True
self.start()
def get_data(self, transceiver, placement, extra_monitor_vertices,
placements):
data = struct.pack("<I", 100)
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=2,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# create socket
connection = UDPConnection(local_host=None, local_port=self.PORT)
# send
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 15, transceiver,
placements,
extra_monitor_vertices)
transceiver.send_sdp_message(message=message)
# receive
output = None
finished = False
first = True
offset = 0
while not finished:
data = connection.receive()
length_of_data = len(data)
if first:
length = struct.unpack_from("<I", data, 0)[0]
first = False
output = bytearray(length)
self._view = memoryview(output)
self._view[offset:offset + length_of_data - 4] = \
data[4:4 + length_of_data - 4]
offset += length_of_data - 4
else:
last_mc_packet = struct.unpack_from("<I", data,
length_of_data - 4)[0]
if last_mc_packet == 0xFFFFFFFF:
self._view[offset:offset + length_of_data - 4] = \
data[0:0 + length_of_data - 4]
offset += length_of_data - 4
finished = True
else:
self._view[offset:offset + length_of_data] = \
data[0:0 + length_of_data]
offset += length_of_data
# hand back
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 4, transceiver,
placements,
extra_monitor_vertices)
return output
def __init__(self):
super(PacketGathererWithProtocol, self).__init__(label="pg",
constraints=None)
self._view = None
self._max_seq_num = None
self._output = None
self._lost_seq_nums = list()
# create socket
self._connection = UDPConnection(local_host=None)
def get_data(self, transceiver, placement, extra_monitor_vertices,
placements):
data = struct.pack("<I", self.SDP_PACKET_START_SENDING_COMMAND_ID)
# print("sending to core {}:{}:{}".format(
# placement.x, placement.y, placement.p))
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# create socket
connection = UDPConnection(local_host=None, local_port=self.PORT)
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 15, transceiver,
placements,
extra_monitor_vertices)
# send message
transceiver.send_sdp_message(message=message)
# receive
finished = False
first = True
seq_num = 1
seq_nums = set()
while not finished:
try:
data = connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
# self._check(seq_nums)
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 4, transceiver,
placements,
extra_monitor_vertices)
connection.close()
return self._output, self._lost_seq_nums
def __init_sender(self, db, vertex_sizes):
if self.__sender_connection is None:
self.__sender_connection = UDPConnection()
for label in self.__send_labels:
self.__send_address_details[label] = self.__get_live_input_details(
db, label)
if self.__machine_vertices:
key, _ = db.get_machine_live_input_key(label)
self._atom_id_to_key[label] = {0: key}
vertex_sizes[label] = 1
else:
self._atom_id_to_key[label] = db.get_atom_id_to_key_mapping(
label)
vertex_sizes[label] = len(self._atom_id_to_key[label])
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockBMP, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection(local_port=SCP_SCAMP_PORT)
self._running = False
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockMachine, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection()
self._running = False
self._messages = deque()
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
class MockMachine(Thread):
""" A Machine that can be used for testing protocol
"""
def __init__(self, responses=None):
"""
:param responses:
An optional list of responses to send in the order to be sent.
If not specified, OK responses will be sent for every request.
Note that responses can include "None" which means that no
response will be sent to that request
"""
super(MockMachine, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection()
self._running = False
self._messages = deque()
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
@property
def is_next_message(self):
return len(self._messages) > 0
@property
def next_message(self):
return self._messages.popleft()
@property
def error(self):
return self._error
@property
def local_port(self):
return self._receiver.local_port
def _do_receive(self):
data, address = self._receiver.receive_with_address()
self._messages.append(data)
sdp_header = SDPHeader.from_bytestring(data, 2)
_result, sequence = struct.unpack_from("<2H", data, 10)
response = None
if self._responses:
response = self._responses.popleft()
response._data = (response._data[:10] +
struct.pack("<H", sequence) +
response._data[12:])
else:
response = _SCPOKMessage(sdp_header.source_chip_x,
sdp_header.source_chip_y, sequence)
if response is not None:
self._receiver.send_to(
struct.pack("<2x") + response.bytestring, address)
def run(self):
self._running = True
while self._running:
try:
if self._receiver.is_ready_to_receive(10):
self._do_receive()
except Exception as e: # pylint: disable=broad-except
if self._running:
traceback.print_exc()
self._error = e
def stop(self):
self._running = False
self._receiver.close()
class MockBMP(Thread):
""" A BMP that can be used for testing protocol
"""
_SCP_HEADER = struct.Struct("<2H")
_TWO_NUL_BYTES = struct.pack("<2x") # Constant byte sequence
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockBMP, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection(local_port=SCP_SCAMP_PORT)
self._running = False
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
@property
def error(self): # pragma: no cover
return self._error
@property
def local_port(self): # pragma: no cover
return self._receiver.local_port
def _do_receive(self):
data, address = self._receiver.receive_with_address(10)
sdp_header = SDPHeader.from_bytestring(data, 2)
_, sequence = self._SCP_HEADER.unpack_from(data, 10)
response = None
if self._responses:
response = self._responses.popleft()
else: # pragma: no cover
response = SCPOKMessage(sdp_header.source_chip_x,
sdp_header.source_chip_y, sequence)
if hasattr(response, "set_sequence"):
response.set_sequence(sequence)
if response is not None:
self._receiver.send_to(self._TWO_NUL_BYTES + response.bytestring,
address)
def run(self):
self._running = True
while self._running:
try:
if self._receiver.is_ready_to_receive():
self._do_receive()
except Exception as e:
if self._running: # pragma: no cover
traceback.print_exc()
self._error = e
def stop(self):
self._running = False
self._receiver.close()
def close(self):
self.__running = False
UDPConnection.close(self)
class MockMachine(Thread):
""" A Machine that can be used for testing protocol
"""
def __init__(self, responses=None):
"""
:param responses:\
An optional list of responses to send in the order to be sent. \
If not specified, OK responses will be sent for every request. \
Note that responses can include "None" which means that no\
response will be sent to that request
"""
super(MockMachine, self).__init__()
# Set up a connection to be the machine
self._receiver = UDPConnection()
self._running = False
self._messages = deque()
self._error = None
self._responses = deque()
if responses is not None:
self._responses.extend(responses)
@property
def is_next_message(self):
return len(self._messages) > 0
@property
def next_message(self):
return self._messages.popleft()
@property
def error(self):
return self._error
@property
def local_port(self):
return self._receiver.local_port
def _do_receive(self):
data, address = self._receiver.receive_with_address()
self._messages.append(data)
sdp_header = SDPHeader.from_bytestring(data, 2)
_result, sequence = struct.unpack_from("<2H", data, 10)
response = None
if self._responses:
response = self._responses.popleft()
response._data = (
response._data[:10] + struct.pack("<H", sequence) +
response._data[12:])
else:
response = _SCPOKMessage(
sdp_header.source_chip_x, sdp_header.source_chip_y,
sequence)
if response is not None:
self._receiver.send_to(
struct.pack("<2x") + response.bytestring, address)
def run(self):
self._running = True
while self._running:
try:
if self._receiver.is_ready_to_receive(10):
self._do_receive()
except Exception as e:
if self._running:
traceback.print_exc()
self._error = e
def stop(self):
self._running = False
self._receiver.close()
class LiveEventConnection(DatabaseConnection):
""" A connection for receiving and sending live events from and to\
SpiNNaker
"""
__slots__ = [
"_atom_id_to_key", "__error_keys", "__init_callbacks",
"__key_to_atom_id_and_label", "__live_event_callbacks",
"__live_packet_gather_label", "__machine_vertices",
"__pause_stop_callbacks", "__receive_labels", "__receiver_connection",
"__receiver_listener", "__send_address_details", "__send_labels",
"__sender_connection", "__start_resume_callbacks"
]
def __init__(self,
live_packet_gather_label,
receive_labels=None,
send_labels=None,
local_host=None,
local_port=NOTIFY_PORT,
machine_vertices=False):
"""
:param str live_packet_gather_label:
The label of the :py:class:`LivePacketGather` vertex to which
received events are being sent
:param iterable(str) receive_labels:
Labels of vertices from which live events will be received.
:param iterable(str) send_labels:
Labels of vertices to which live events will be sent
:param str local_host:
Optional specification of the local hostname or IP address of the
interface to listen on
:param int local_port:
Optional specification of the local port to listen on. Must match
the port that the toolchain will send the notification on (19999
by default)
"""
# pylint: disable=too-many-arguments
super(LiveEventConnection, self).__init__(self.__do_start_resume,
self.__do_stop_pause,
local_host=local_host,
local_port=local_port)
self.add_database_callback(self.__read_database_callback)
self.__live_packet_gather_label = live_packet_gather_label
self.__receive_labels = (list(receive_labels)
if receive_labels is not None else None)
self.__send_labels = (list(send_labels)
if send_labels is not None else None)
self.__machine_vertices = machine_vertices
self.__sender_connection = None
self.__send_address_details = dict()
# Also used by SpynnakerPoissonControlConnection
self._atom_id_to_key = dict()
self.__key_to_atom_id_and_label = dict()
self.__live_event_callbacks = list()
self.__start_resume_callbacks = dict()
self.__pause_stop_callbacks = dict()
self.__init_callbacks = dict()
if receive_labels is not None:
for label in receive_labels:
self.__live_event_callbacks.append(list())
self.__start_resume_callbacks[label] = list()
self.__pause_stop_callbacks[label] = list()
self.__init_callbacks[label] = list()
if send_labels is not None:
for label in send_labels:
self.__start_resume_callbacks[label] = list()
self.__pause_stop_callbacks[label] = list()
self.__init_callbacks[label] = list()
self.__receiver_listener = None
self.__receiver_connection = None
self.__error_keys = set()
def add_send_label(self, label):
if self.__send_labels is None:
self.__send_labels = list()
if label not in self.__send_labels:
self.__send_labels.append(label)
if label not in self.__start_resume_callbacks:
self.__start_resume_callbacks[label] = list()
self.__pause_stop_callbacks[label] = list()
self.__init_callbacks[label] = list()
def add_receive_label(self, label):
if self.__receive_labels is None:
self.__receive_labels = list()
if label not in self.__receive_labels:
self.__receive_labels.append(label)
self.__live_event_callbacks.append(list())
if label not in self.__start_resume_callbacks:
self.__start_resume_callbacks[label] = list()
self.__pause_stop_callbacks[label] = list()
self.__init_callbacks[label] = list()
def add_init_callback(self, label, init_callback):
""" Add a callback to be called to initialise a vertex
:param str label:
The label of the vertex to be notified about. Must be one of the
vertices listed in the constructor
:param init_callback: A function to be called to initialise the\
vertex. This should take as parameters the label of the vertex,\
the number of neurons in the population, the run time of the\
simulation in milliseconds, and the simulation timestep in\
milliseconds
:type init_callback: callable(str, int, float, float) -> None
"""
self.__init_callbacks[label].append(init_callback)
def add_receive_callback(self,
label,
live_event_callback,
translate_key=True):
""" Add a callback for the reception of live events from a vertex
:param str label: The label of the vertex to be notified about.
Must be one of the vertices listed in the constructor
:param live_event_callback: A function to be called when events are\
received. This should take as parameters the label of the vertex,\
the simulation timestep when the event occurred, and an\
array-like of atom IDs.
:type live_event_callback: callable(str, int, list(int)) -> None
:param bool translate_key:
True if the key is to be converted to an atom ID, False if the
key should stay a key
"""
label_id = self.__receive_labels.index(label)
logger.info("Receive callback {} registered to label {}".format(
live_event_callback, label))
self.__live_event_callbacks[label_id].append(
(live_event_callback, translate_key))
def add_start_callback(self, label, start_callback):
""" Add a callback for the start of the simulation
:param start_callback: A function to be called when the start\
message has been received. This function should take the label of\
the referenced vertex, and an instance of this class, which can\
be used to send events
:type start_callback: callable(str, LiveEventConnection) -> None
:param str label: the label of the function to be sent
"""
logger.warning(
"the method 'add_start_callback(label, start_callback)' is in "
"deprecation, and will be replaced with the method "
"'add_start_resume_callback(label, start_resume_callback)' in a "
"future release.")
self.add_start_resume_callback(label, start_callback)
def add_start_resume_callback(self, label, start_resume_callback):
""" Add a callback for the start and resume state of the simulation
:param str label: the label of the function to be sent
:param start_resume_callback: A function to be called when the start\
or resume message has been received. This function should take \
the label of the referenced vertex, and an instance of this \
class, which can be used to send events.
:type start_resume_callback: callable(str, LiveEventConnection) -> None
:rtype: None
"""
self.__start_resume_callbacks[label].append(start_resume_callback)
def add_pause_stop_callback(self, label, pause_stop_callback):
""" Add a callback for the pause and stop state of the simulation
:param str label: the label of the function to be sent
:param pause_stop_callback: A function to be called when the pause\
or stop message has been received. This function should take the\
label of the referenced vertex, and an instance of this class,\
which can be used to send events.
:type pause_stop_callback: callable(str, LiveEventConnection) -> None
:rtype: None
"""
self.__pause_stop_callbacks[label].append(pause_stop_callback)
def __read_database_callback(self, db_reader):
self.__handle_possible_rerun_state()
vertex_sizes = OrderedDict()
run_time_ms = db_reader.get_configuration_parameter_value("runtime")
machine_timestep_ms = db_reader.get_configuration_parameter_value(
"machine_time_step") / 1000.0
if self.__send_labels is not None:
self.__init_sender(db_reader, vertex_sizes)
if self.__receive_labels is not None:
self.__init_receivers(db_reader, vertex_sizes)
for label, vertex_size in iteritems(vertex_sizes):
for init_callback in self.__init_callbacks[label]:
init_callback(label, vertex_size, run_time_ms,
machine_timestep_ms)
def __init_sender(self, db, vertex_sizes):
if self.__sender_connection is None:
self.__sender_connection = UDPConnection()
for label in self.__send_labels:
self.__send_address_details[label] = self.__get_live_input_details(
db, label)
if self.__machine_vertices:
key, _ = db.get_machine_live_input_key(label)
self._atom_id_to_key[label] = {0: key}
vertex_sizes[label] = 1
else:
self._atom_id_to_key[label] = db.get_atom_id_to_key_mapping(
label)
vertex_sizes[label] = len(self._atom_id_to_key[label])
def __init_receivers(self, db, vertex_sizes):
# Set up a single connection for receive
if self.__receiver_connection is None:
self.__receiver_connection = EIEIOConnection()
receivers = set()
for label_id, label in enumerate(self.__receive_labels):
_, port, board_address, tag = self.__get_live_output_details(
db, label)
# Update the tag if not already done
if (board_address, port, tag) not in receivers:
self.__update_tag(self.__receiver_connection, board_address,
tag)
receivers.add((board_address, port, tag))
send_port_trigger_message(self.__receiver_connection,
board_address)
logger.info("Listening for traffic from {} on {}:{}", label,
self.__receiver_connection.local_ip_address,
self.__receiver_connection.local_port)
if self.__machine_vertices:
key, _ = db.get_machine_live_output_key(
label, self.__live_packet_gather_label)
self.__key_to_atom_id_and_label[key] = (0, label_id)
vertex_sizes[label] = 1
else:
key_to_atom_id = db.get_key_to_atom_id_mapping(label)
for key, atom_id in iteritems(key_to_atom_id):
self.__key_to_atom_id_and_label[key] = (atom_id, label_id)
vertex_sizes[label] = len(key_to_atom_id)
# Last of all, set up the listener for packets
# NOTE: Has to be done last as otherwise will receive SCP messages
# sent above!
if self.__receiver_listener is None:
self.__receiver_listener = ConnectionListener(
self.__receiver_connection)
self.__receiver_listener.add_callback(self.__do_receive_packet)
self.__receiver_listener.start()
def __get_live_input_details(self, db_reader, send_label):
if self.__machine_vertices:
x, y, p = db_reader.get_placement(send_label)
else:
x, y, p = db_reader.get_placements(send_label)[0]
ip_address = db_reader.get_ip_address(x, y)
return x, y, p, ip_address
def __get_live_output_details(self, db_reader, receive_label):
if self.__machine_vertices:
host, port, strip_sdp, board_address, tag = \
db_reader.get_machine_live_output_details(
receive_label, self.__live_packet_gather_label)
if host is None:
raise Exception(
"no live output tag found for {} in machine graph".format(
receive_label))
else:
host, port, strip_sdp, board_address, tag = \
db_reader.get_live_output_details(
receive_label, self.__live_packet_gather_label)
if host is None:
raise Exception(
"no live output tag found for {} in app graph".format(
receive_label))
if not strip_sdp:
raise Exception("Currently, only IP tags which strip the SDP "
"headers are supported")
return host, port, board_address, tag
def __update_tag(self, connection, board_address, tag):
# Update an IP Tag with the sender's address and port
# This avoids issues with NAT firewalls
logger.debug("Updating tag for {}".format(board_address))
request = IPTagSet(0,
0, [0, 0, 0, 0],
0,
tag,
strip=True,
use_sender=True)
request.sdp_header.flags = SDPFlag.REPLY_EXPECTED_NO_P2P
update_sdp_header_for_udp_send(request.sdp_header, 0, 0)
data = _TWO_SKIP.pack() + request.bytestring
sent = False
tries_to_go = 3
while not sent:
try:
connection.send_to(data, (board_address, SCP_SCAMP_PORT))
response_data = connection.receive(1.0)
request.get_scp_response().read_bytestring(
response_data, _TWO_SKIP.size)
sent = True
except SpinnmanTimeoutException:
if not tries_to_go:
logger.info("No more tries - Error!")
reraise(*sys.exc_info())
logger.info("Timeout, retrying")
tries_to_go -= 1
logger.debug("Done updating tag for {}".format(board_address))
def __handle_possible_rerun_state(self):
# reset from possible previous calls
if self.__sender_connection is not None:
self.__sender_connection.close()
self.__sender_connection = None
if self.__receiver_listener is not None:
self.__receiver_listener.close()
self.__receiver_listener = None
if self.__receiver_connection is not None:
self.__receiver_connection.close()
self.__receiver_connection = None
def __launch_thread(self, kind, label, callback):
thread = Thread(
target=callback,
args=(label, self),
name="{} callback thread for live_event_connection {}:{}".format(
kind, self._local_port, self._local_ip_address))
thread.start()
def __do_start_resume(self):
for label, callbacks in iteritems(self.__start_resume_callbacks):
for callback in callbacks:
self.__launch_thread("start_resume", label, callback)
def __do_stop_pause(self):
for label, callbacks in iteritems(self.__pause_stop_callbacks):
for callback in callbacks:
self.__launch_thread("pause_stop", label, callback)
def __do_receive_packet(self, packet):
# pylint: disable=broad-except
logger.debug("Received packet")
try:
if packet.eieio_header.is_time:
self.__handle_time_packet(packet)
else:
self.__handle_no_time_packet(packet)
except Exception:
logger.warning("problem handling received packet", exc_info=True)
def __handle_time_packet(self, packet):
key_times_labels = OrderedDict()
atoms_times_labels = OrderedDict()
while packet.is_next_element:
element = packet.next_element
time = element.payload
key = element.key
if key in self.__key_to_atom_id_and_label:
atom_id, label_id = self.__key_to_atom_id_and_label[key]
if time not in key_times_labels:
key_times_labels[time] = dict()
atoms_times_labels[time] = dict()
if label_id not in key_times_labels[time]:
key_times_labels[time][label_id] = list()
atoms_times_labels[time][label_id] = list()
key_times_labels[time][label_id].append(key)
atoms_times_labels[time][label_id].append(atom_id)
else:
self.__handle_unknown_key(key)
for time in iterkeys(key_times_labels):
for label_id in iterkeys(key_times_labels[time]):
label = self.__receive_labels[label_id]
for c_back, use_atom in self.__live_event_callbacks[label_id]:
if use_atom:
c_back(label, time, atoms_times_labels[time][label_id])
else:
c_back(label, time, key_times_labels[time][label_id])
def __handle_no_time_packet(self, packet):
while packet.is_next_element:
element = packet.next_element
key = element.key
if key in self.__key_to_atom_id_and_label:
atom_id, label_id = self.__key_to_atom_id_and_label[key]
label = self.__receive_labels[label_id]
for c_back, use_atom in self.__live_event_callbacks[label_id]:
if isinstance(element, KeyPayloadDataElement):
if use_atom:
c_back(label, atom_id, element.payload)
else:
c_back(label, key, element.payload)
else:
if use_atom:
c_back(label, atom_id)
else:
c_back(label, key)
else:
self.__handle_unknown_key(key)
def __handle_unknown_key(self, key):
if key not in self.__error_keys:
self.__error_keys.add(key)
logger.warning("Received unexpected key {}".format(key))
def send_event(self, label, atom_id, send_full_keys=False):
""" Send an event from a single atom
:param str label:
The label of the vertex from which the event will originate
:param int atom_id: The ID of the atom sending the event
:param bool send_full_keys:
Determines whether to send full 32-bit keys, getting the key for
each atom from the database, or whether to send 16-bit atom IDs
directly
"""
self.send_events(label, [atom_id], send_full_keys)
def send_events(self, label, atom_ids, send_full_keys=False):
""" Send a number of events
:param str label:
The label of the vertex from which the events will originate
:param list(int) atom_ids: array-like of atom IDs sending events
:param bool send_full_keys:
Determines whether to send full 32-bit keys, getting the key for
each atom from the database, or whether to send 16-bit atom IDs
directly
"""
max_keys = _MAX_HALF_KEYS_PER_PACKET
msg_type = EIEIOType.KEY_16_BIT
if send_full_keys:
max_keys = _MAX_FULL_KEYS_PER_PACKET
msg_type = EIEIOType.KEY_32_BIT
pos = 0
x, y, p, ip_address = self.__send_address_details[label]
while pos < len(atom_ids):
message = EIEIODataMessage.create(msg_type)
events_in_packet = 0
while pos < len(atom_ids) and events_in_packet < max_keys:
key = atom_ids[pos]
if send_full_keys:
key = self._atom_id_to_key[label][key]
message.add_key(key)
pos += 1
events_in_packet += 1
self.__sender_connection.send_to(
self.__get_sdp_data(message, x, y, p),
(ip_address, SCP_SCAMP_PORT))
def send_event_with_payload(self, label, atom_id, payload):
""" Send an event with a payload from a single atom
:param str label:
The label of the vertex from which the event will originate
:param int atom_id: The ID of the atom sending the event
:param int payload: The payload to send
"""
self.send_events_with_payloads(label, [(atom_id, payload)])
def send_events_with_payloads(self, label, atom_ids_and_payloads):
""" Send a number of events with payloads
:param str label:
The label of the vertex from which the events will originate
:param list(tuple(int,int)) atom_ids_and_payloads:
array-like of tuples of atom IDs sending events with their payloads
"""
msg_type = EIEIOType.KEY_PAYLOAD_32_BIT
max_keys = _MAX_FULL_KEYS_PAYLOADS_PER_PACKET
pos = 0
x, y, p, ip_address = self.__send_address_details[label]
while pos < len(atom_ids_and_payloads):
message = EIEIODataMessage.create(msg_type)
events = 0
while pos < len(atom_ids_and_payloads) and events < max_keys:
key, payload = atom_ids_and_payloads[pos]
key = self._atom_id_to_key[label][key]
message.add_key_and_payload(key, payload)
pos += 1
events += 1
self.__sender_connection.send_to(
self.__get_sdp_data(message, x, y, p),
(ip_address, SCP_SCAMP_PORT))
def send_eieio_message(self, message, label):
""" Send an EIEIO message (using one-way the live input) to the \
vertex with the given label.
:param ~spinnman.messages.eieio.AbstractEIEIOMessage message:
The EIEIO message to send
:param str label: The label of the receiver machine vertex
"""
target = self.__send_address_details[label]
if target is None:
return
x, y, p, ip_address = target
self.__sender_connection.send_to(self.__get_sdp_data(message, x, y, p),
(ip_address, SCP_SCAMP_PORT))
def close(self):
self.__handle_possible_rerun_state()
super(LiveEventConnection, self).close()
@staticmethod
def __get_sdp_data(message, x, y, p):
# Create an SDP message - no reply so source is unimportant
# SDP port can be anything except 0 as the target doesn't care
sdp_message = SDPMessage(SDPHeader(flags=SDPFlag.REPLY_NOT_EXPECTED,
tag=0,
destination_port=1,
destination_cpu=p,
destination_chip_x=x,
destination_chip_y=y,
source_port=0,
source_cpu=0,
source_chip_x=0,
source_chip_y=0),
data=message.bytestring)
return _TWO_SKIP.pack() + sdp_message.bytestring
def get_data(self, transceiver, placement):
data = struct.pack("<I", self.SDP_PACKET_START_SENDING_COMMAND_ID)
# print("sending to core {}:{}:{}".format(
# placement.x, placement.y, placement.p))
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# create socket
connection = UDPConnection(local_host=None, local_port=self.PORT)
# send
transceiver.set_reinjection_router_timeout(15, 15)
transceiver.send_sdp_message(message=message)
# receive
finished = False
first = True
seq_num = 1
seq_nums = set()
while not finished:
try:
data = connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
# pretend that we're broken, re-require some of the data
print("doing fake retransmission")
missing_seq_nums = [3140, 1938]
self._remove_seq_nums(seq_nums, missing_seq_nums)
finished = self._transmit_missing_seq_nums(seq_nums, transceiver,
placement)
while not finished:
try:
data = connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
# self._check(seq_nums)
transceiver.set_reinjection_router_timeout(15, 4)
return self._output
class PacketGathererWithProtocol(MachineVertex, MachineDataSpecableVertex,
AbstractHasAssociatedBinary):
DATA_REGIONS = Enum(value="DATA_REGIONS",
names=[('SYSTEM', 0), ('CONFIG', 1)])
SDRAM_READING_SIZE_IN_BYTES_CONVERTER = 1024 * 1024
CONFIG_SIZE = 8
DATA_PER_FULL_PACKET = 68 # 272 bytes as removed scp header
DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM = DATA_PER_FULL_PACKET - 1
WORD_TO_BYTE_CONVERTER = 4
TIMEOUT_PER_RECEIVE_IN_SECONDS = 1
TIME_OUT_FOR_SENDING_IN_SECONDS = 0.01
SDP_PACKET_START_SENDING_COMMAND_ID = 100
SDP_PACKET_START_MISSING_SEQ_COMMAND_ID = 1000
SDP_PACKET_MISSING_SEQ_COMMAND_ID = 1001
SDP_PACKET_PORT = 2
SDP_RETRANSMISSION_HEADER_SIZE = 2
END_FLAG = 0xFFFFFFFF
END_FLAG_SIZE = 4
SEQUENCE_NUMBER_SIZE = 4
N_PACKETS_SIZE = 4
LENGTH_OF_DATA_SIZE = 4
def __init__(self):
super(PacketGathererWithProtocol, self).__init__(label="pg",
constraints=None)
self._view = None
self._max_seq_num = None
self._output = None
self._lost_seq_nums = list()
# create socket
self._connection = UDPConnection(local_host=None)
@property
def resources_required(self):
return ResourceContainer(
sdram=SDRAMResource(constants.SYSTEM_BYTES_REQUIREMENT +
self.CONFIG_SIZE),
iptags=[
IPtagResource(port=self._connection.local_port,
strip_sdp=True,
ip_address="localhost")
])
def get_binary_start_type(self):
return ExecutableType.USES_SIMULATION_INTERFACE
def generate_machine_data_specification(self, spec, placement,
machine_graph, routing_info,
iptags, reverse_iptags,
machine_time_step,
time_scale_factor):
# Setup words + 1 for flags + 1 for recording size
setup_size = constants.SYSTEM_BYTES_REQUIREMENT
# Create the data regions for hello world
self._reserve_memory_regions(spec, setup_size)
# write data for the simulation data item
spec.switch_write_focus(self.DATA_REGIONS.SYSTEM.value)
spec.write_array(
simulation_utilities.get_simulation_header_array(
self.get_binary_file_name(), machine_time_step,
time_scale_factor))
spec.switch_write_focus(self.DATA_REGIONS.CONFIG.value)
base_key = routing_info.get_first_key_for_edge(
list(machine_graph.get_edges_ending_at_vertex(self))[0])
spec.write_value(base_key + 1)
spec.write_value(base_key + 2)
# End-of-Spec:
spec.end_specification()
def _reserve_memory_regions(self, spec, system_size):
spec.reserve_memory_region(region=self.DATA_REGIONS.SYSTEM.value,
size=system_size,
label='systemInfo')
spec.reserve_memory_region(region=self.DATA_REGIONS.CONFIG.value,
size=self.CONFIG_SIZE,
label="config")
def get_binary_file_name(self):
return "packet_gatherer.aplx"
def get_data(self, transceiver, placement, extra_monitor_vertices,
placements):
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 15, transceiver,
placements,
extra_monitor_vertices)
# spur off a c code version
subprocess.call(("host_"))
data = struct.pack("<I", self.SDP_PACKET_START_SENDING_COMMAND_ID)
# print("sending to core {}:{}:{}".format(
# placement.x, placement.y, placement.p))
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# send
transceiver.send_sdp_message(message=message)
# receive
finished = False
first = True
seq_num = 1
seq_nums = set()
while not finished:
try:
data = self._connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
# self._check(seq_nums)
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 4, transceiver,
placements,
extra_monitor_vertices)
return self._output, self._lost_seq_nums
def _calculate_missing_seq_nums(self, seq_nums):
missing_seq_nums = list()
for seq_num in range(1, self._max_seq_num):
if seq_num not in seq_nums:
missing_seq_nums.append(seq_num)
return missing_seq_nums
def _transmit_missing_seq_nums(self, seq_nums, transceiver, placement):
# locate missing seq nums from pile
missing_seq_nums = self._calculate_missing_seq_nums(seq_nums)
self._lost_seq_nums.append(len(missing_seq_nums))
# self._print_missing(seq_nums)
if len(missing_seq_nums) == 0:
return True
# print("doing retransmission")
# figure n packets given the 2 formats
n_packets = 1
length_via_format2 = \
len(missing_seq_nums) - (self.DATA_PER_FULL_PACKET - 2)
if length_via_format2 > 0:
n_packets += int(
math.ceil(
float(length_via_format2) /
float(self.DATA_PER_FULL_PACKET - 1)))
# transmit missing seq as a new sdp packet
first = True
seq_num_offset = 0
for _packet_count in range(0, n_packets):
length_left_in_packet = self.DATA_PER_FULL_PACKET
offset = 0
data = None
size_of_data_left_to_transmit = None
# if first, add n packets to list
if first:
# get left over space / data size
size_of_data_left_to_transmit = min(
length_left_in_packet - 2,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 2) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag and n packets
struct.pack_into("<I", data, offset,
self.SDP_PACKET_START_MISSING_SEQ_COMMAND_ID)
struct.pack_into("<I", data, self.WORD_TO_BYTE_CONVERTER,
n_packets)
# update state
offset += 2 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 2
first = False
else: # just add data
# get left over space / data size
size_of_data_left_to_transmit = min(
self.DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 1) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag
struct.pack_into("<I", data, offset,
self.SDP_PACKET_MISSING_SEQ_COMMAND_ID)
offset += 1 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 1
# fill data field
struct.pack_into(
"<{}I".format(size_of_data_left_to_transmit), data, offset,
*missing_seq_nums[seq_num_offset:seq_num_offset +
size_of_data_left_to_transmit])
seq_num_offset += length_left_in_packet
# build sdp message
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# debug
# self._print_out_packet_data(data)
# send message to core
transceiver.send_sdp_message(message=message)
# sleep for ensuring core doesnt lose packets
time.sleep(self.TIME_OUT_FOR_SENDING_IN_SECONDS)
# self._print_packet_num_being_sent(packet_count, n_packets)
return False
def _process_data(self, data, first, seq_num, seq_nums, finished,
placement, transceiver):
# self._print_out_packet_data(data)
length_of_data = len(data)
if first:
length = struct.unpack_from("<I", data, 0)[0]
first = False
self._output = bytearray(length)
self._view = memoryview(self._output)
self._write_into_view(0, length_of_data - self.LENGTH_OF_DATA_SIZE,
data, self.LENGTH_OF_DATA_SIZE,
length_of_data, seq_num, length_of_data,
False)
# deduce max seq num for future use
self._max_seq_num = self.calculate_max_seq_num()
else: # some data packet
first_packet_element = struct.unpack_from("<I", data, 0)[0]
last_mc_packet = struct.unpack_from(
"<I", data, length_of_data - self.END_FLAG_SIZE)[0]
# if received a last flag on its own, its during retransmission.
# check and try again if required
if (last_mc_packet == self.END_FLAG
and length_of_data == self.END_FLAG_SIZE):
if not self._check(seq_nums):
finished = self._transmit_missing_seq_nums(
placement=placement,
transceiver=transceiver,
seq_nums=seq_nums)
else:
# this flag can be dropped at some point
seq_num = first_packet_element
# print("seq num = {}".format(seq_num))
if seq_num > self._max_seq_num:
raise Exception(
"got an insane sequence number. got {} when "
"the max is {} with a length of {}".format(
seq_num, self._max_seq_num, length_of_data))
seq_nums.add(seq_num)
# figure offset for where data is to be put
offset = self._calculate_offset(seq_num)
# write excess data as required
if last_mc_packet == self.END_FLAG:
# adjust for end flag
true_data_length = (length_of_data - self.END_FLAG_SIZE -
self.SEQUENCE_NUMBER_SIZE)
# write data
self._write_into_view(offset, offset + true_data_length,
data, self.SEQUENCE_NUMBER_SIZE,
length_of_data - self.END_FLAG_SIZE,
seq_num, length_of_data, True)
# check if need to retry
if not self._check(seq_nums):
finished = self._transmit_missing_seq_nums(
placement=placement,
transceiver=transceiver,
seq_nums=seq_nums)
else:
finished = True
else: # full block of data, just write it in
true_data_length = (offset + length_of_data -
self.SEQUENCE_NUMBER_SIZE)
self._write_into_view(offset, true_data_length, data,
self.SEQUENCE_NUMBER_SIZE,
length_of_data, seq_num,
length_of_data, False)
return first, seq_num, seq_nums, finished
def _calculate_offset(self, seq_num):
offset = (seq_num * self.DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM *
self.WORD_TO_BYTE_CONVERTER)
return offset
def _write_into_view(self, view_start_position, view_end_position, data,
data_start_position, data_end_position, seq_num,
packet_length, is_final):
""" puts data into the view
:param view_start_position: where in view to start
:param view_end_position: where in view to end
:param data: the data holder to write from
:param data_start_position: where in data holder to start from
:param data_end_position: where in data holder to end
:param seq_num: the seq number to figure
:rtype: None
"""
if view_end_position > len(self._output):
raise Exception(
"I'm trying to add to my output data, but am trying to add "
"outside my acceptable output positions!!!! max is {} and "
"I received request to fill to {} for seq num {} from max "
"seq num {} length of packet {} and final {}".format(
len(self._output), view_end_position, seq_num,
self._max_seq_num, packet_length, is_final))
# print("view_start={} view_end={} data_start={} data_end={}".format(
# view_start_position, view_end_position, data_start_position,
# data_end_position))
self._view[view_start_position: view_end_position] = \
data[data_start_position:data_end_position]
def _check(self, seq_nums):
# hand back
seq_nums = sorted(seq_nums)
max_needed = self.calculate_max_seq_num()
if len(seq_nums) > max_needed:
raise Exception("I've received more data than i was expecting!!")
if len(seq_nums) != max_needed:
# self._print_length_of_received_seq_nums(seq_nums, max_needed)
return False
return True
def calculate_max_seq_num(self):
n_sequence_numbers = 0
data_left = len(self._output) - (
(self.DATA_PER_FULL_PACKET - self.SDP_RETRANSMISSION_HEADER_SIZE) *
self.WORD_TO_BYTE_CONVERTER)
extra_n_sequences = float(data_left) / float(
self.DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM *
self.WORD_TO_BYTE_CONVERTER)
n_sequence_numbers += math.ceil(extra_n_sequences)
return int(n_sequence_numbers)
@staticmethod
def _print_missing(seq_nums):
last_seq_num = 0
seq_nums = sorted(seq_nums)
for seq_num in seq_nums:
if seq_num != last_seq_num + 1:
print("from list im missing seq num {}".format(seq_num))
last_seq_num = seq_num
def _print_out_packet_data(self, data):
reread_data = struct.unpack(
"<{}I".format(
int(math.ceil(len(data) / self.WORD_TO_BYTE_CONVERTER))), data)
print(
"converted data back into readable form is {}".format(reread_data))
@staticmethod
def _print_length_of_received_seq_nums(seq_nums, max_needed):
if len(seq_nums) != max_needed:
print("should have received {} sequence numbers, but received "
"{} sequence numbers".format(max_needed, len(seq_nums)))
return False
return True
@staticmethod
def _print_packet_num_being_sent(packet_count, n_packets):
print("send sdp packet with missing seq nums: {} of {}".format(
packet_count + 1, n_packets))