def __init__(self, x, y):
scp_header = SCPRequestHeader(command=SCPResult.RC_OK)
sdp_header = SDPHeader(flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_port=0,
destination_cpu=0,
destination_chip_x=x,
destination_chip_y=y)
utils.update_sdp_header_for_udp_send(sdp_header, 0, 0)
SDPMessage.__init__(self, sdp_header, data=scp_header.bytestring)
def send_output_to_spinnaker(self, value, placement, transceiver):
# Apply the pre-slice, the connection function and the transform.
c_value = value[(self._managing_app_outgoing_partition.identifier.
transmission_parameter.pre_slice)]
# locate required transforms and functions
partition_transmission_function = \
self._managing_app_outgoing_partition.identifier\
.transmission_parameter.parameter_function
partition_transmission_transform = \
self._managing_app_outgoing_partition.identifier\
.transmission_parameter.full_transform(slice_out=False)
# execute function if required
if partition_transmission_function is not None:
c_value = partition_transmission_function(c_value)
# do transform
c_value = numpy.dot(partition_transmission_transform, c_value)
# create SCP packet
# c_value is converted to S16.15
data = helpful_functions.convert_numpy_array_to_s16_15(c_value)
packet = SDPMessage(sdp_header=SDPHeader(
destination_port=constants.SDP_PORTS.SDP_RECEIVER.value,
destination_cpu=placement.p,
destination_chip_x=placement.x,
destination_chip_y=placement.y,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=bytes(data.data))
transceiver.send_sdp_message(packet)
self._n_packets_transmitted += 1
def __call__(self, txrx, app_id, all_core_subsets):
# check that the right number of processors are in sync
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
total_processors = len(all_core_subsets)
left_to_do_cores = total_processors - processors_completed
progress = ProgressBar(
left_to_do_cores,
"Forcing error cores to generate provenance data")
error_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.RUN_TIME_EXCEPTION)
watchdog_cores = txrx.get_cores_in_state(all_core_subsets,
CPUState.WATCHDOG)
idle_cores = txrx.get_cores_in_state(all_core_subsets, CPUState.IDLE)
if (len(error_cores) != 0 or len(watchdog_cores) != 0
or len(idle_cores) != 0):
raise ConfigurationException(
"Some cores have crashed. RTE cores {}, watch-dogged cores {},"
" idle cores {}".format(error_cores.values(),
watchdog_cores.values(),
idle_cores.values()))
# check that all cores are in the state FINISHED which shows that
# the core has received the message and done provenance updating
attempts = 0
while processors_completed != total_processors and attempts < 10:
attempts += 1
unsuccessful_cores = txrx.get_cores_not_in_state(
all_core_subsets, CPUState.FINISHED)
for (x, y, p) in unsuccessful_cores.iterkeys():
data = struct.pack(
"<I", SDP_RUNNING_MESSAGE_CODES.
SDP_UPDATE_PROVENCE_REGION_AND_EXIT.value)
txrx.send_sdp_message(
SDPMessage(SDPHeader(flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_cpu=p,
destination_chip_x=x,
destination_port=SDP_PORTS.
RUNNING_COMMAND_SDP_PORT.value,
destination_chip_y=y),
data=data))
processors_completed = txrx.get_core_state_count(
app_id, CPUState.FINISHED)
left_over_now = total_processors - processors_completed
to_update = left_to_do_cores - left_over_now
left_to_do_cores = left_over_now
if to_update != 0:
progress.update(to_update)
if attempts >= 10:
logger.error("Unable to Finish getting provenance data. "
"Abandoned after too many retries. "
"Board may be left in an unstable state!")
progress.end()
def get_data(self, transceiver, placement, extra_monitor_vertices,
placements):
data = struct.pack("<I", 100)
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=2,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# create socket
connection = UDPConnection(local_host=None, local_port=self.PORT)
# send
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 15, transceiver,
placements,
extra_monitor_vertices)
transceiver.send_sdp_message(message=message)
# receive
output = None
finished = False
first = True
offset = 0
while not finished:
data = connection.receive()
length_of_data = len(data)
if first:
length = struct.unpack_from("<I", data, 0)[0]
first = False
output = bytearray(length)
self._view = memoryview(output)
self._view[offset:offset + length_of_data - 4] = \
data[4:4 + length_of_data - 4]
offset += length_of_data - 4
else:
last_mc_packet = struct.unpack_from("<I", data,
length_of_data - 4)[0]
if last_mc_packet == 0xFFFFFFFF:
self._view[offset:offset + length_of_data - 4] = \
data[0:0 + length_of_data - 4]
offset += length_of_data - 4
finished = True
else:
self._view[offset:offset + length_of_data] = \
data[0:0 + length_of_data]
offset += length_of_data
# hand back
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 4, transceiver,
placements,
extra_monitor_vertices)
return output
def _retrieve_and_store_data(self, packet):
""" Following a SpinnakerRequestReadData packet, the data stored\
during the simulation needs to be read by the host and stored in\
a data structure, following the specifications of buffering out\
technique.
:param packet: SpinnakerRequestReadData packet received from the\
SpiNNaker system
:type packet:\
:py:class:`spinnman.messages.eieio.command_messages.spinnaker_request_read_data.SpinnakerRequestReadData`
:rtype: None
"""
x = packet.x
y = packet.y
p = packet.p
# check packet sequence number
pkt_seq = packet.sequence_no
last_pkt_seq = self._received_data.last_sequence_no_for_core(x, y, p)
next_pkt_seq = (last_pkt_seq + 1) % 256
if pkt_seq != next_pkt_seq:
# this sequence number is incorrect
# re-sent last HostDataRead packet sent
last_packet_sent = self._received_data.last_sent_packet_to_core(
x, y, p)
if last_packet_sent is None:
raise Exception(
"{}, {}, {}: Something somewhere went terribly wrong - "
"The packet sequence numbers have gone wrong somewhere: "
"the packet sent from the board has incorrect sequence "
"number, but the host never sent one acknowledge".format(
x, y, p))
self._transceiver.send_sdp_message(last_packet_sent)
return
# read data from memory, store it and create data for return ACK packet
ack_packet = self._assemble_ack_packet(x, y, p, packet, pkt_seq)
# create SDP header and message
return_message = SDPMessage(
SDPHeader(
destination_port=SDP_PORTS.OUTPUT_BUFFERING_SDP_PORT.value,
destination_cpu=p,
destination_chip_x=x,
destination_chip_y=y,
flags=SDPFlag.REPLY_NOT_EXPECTED), ack_packet.bytestring)
# storage of last packet received
self._received_data.store_last_received_packet_from_core(
x, y, p, packet)
self._received_data.update_sequence_no_for_core(x, y, p, pkt_seq)
# store last sent message and send to the appropriate core
self._received_data.store_last_sent_packet_to_core(
x, y, p, return_message)
self._transceiver.send_sdp_message(return_message)
def _send_chip_update_provenance_and_exit(txrx, x, y, p):
cmd = SDP_RUNNING_MESSAGE_CODES.SDP_UPDATE_PROVENCE_REGION_AND_EXIT
port = SDP_PORTS.RUNNING_COMMAND_SDP_PORT
txrx.send_sdp_message(
SDPMessage(SDPHeader(flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_port=port.value,
destination_cpu=p,
destination_chip_x=x,
destination_chip_y=y),
data=_ONE_WORD.pack(cmd.value)))
def _update_provenance_and_exit(txrx, processor, core_subset):
byte_data = _ONE_WORD.pack(SDP_RUNNING_MESSAGE_CODES.
SDP_UPDATE_PROVENCE_REGION_AND_EXIT.value)
txrx.send_sdp_message(
SDPMessage(sdp_header=SDPHeader(
flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_port=SDP_PORTS.RUNNING_COMMAND_SDP_PORT.value,
destination_cpu=processor,
destination_chip_x=core_subset.x,
destination_chip_y=core_subset.y),
data=byte_data))
def get_data(self, transceiver, placement, extra_monitor_vertices,
placements):
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 15, transceiver,
placements,
extra_monitor_vertices)
# spur off a c code version
subprocess.call(("host_"))
data = struct.pack("<I", self.SDP_PACKET_START_SENDING_COMMAND_ID)
# print("sending to core {}:{}:{}".format(
# placement.x, placement.y, placement.p))
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# send
transceiver.send_sdp_message(message=message)
# receive
finished = False
first = True
seq_num = 1
seq_nums = set()
while not finished:
try:
data = self._connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
# self._check(seq_nums)
# set router time out
extra_monitor_vertices[0].set_router_time_outs(15, 4, transceiver,
placements,
extra_monitor_vertices)
return self._output, self._lost_seq_nums
def __request_read_data(self, packet):
if not self._finished:
# Send an ACK message to stop the core sending more messages
ack_message_header = SDPHeader(
destination_port=(SDP_PORTS.OUTPUT_BUFFERING_SDP_PORT.value),
destination_cpu=packet.p,
destination_chip_x=packet.x,
destination_chip_y=packet.y,
flags=SDPFlag.REPLY_NOT_EXPECTED)
ack_message_data = HostDataReadAck(packet.sequence_no)
ack_message = SDPMessage(ack_message_header,
ack_message_data.bytestring)
self._transceiver.send_sdp_message(ack_message)
self._buffering_out_thread_pool.apply_async(
self._process_buffered_in_packet, args=[packet])
def _send_request(self, vertex, message):
""" Sends a request
:param vertex: The vertex to send to
:param message: The message to send
"""
placement = self._placements.get_placement_of_vertex(vertex)
sdp_header = SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
flags=SDPFlag.REPLY_NOT_EXPECTED,
destination_port=SDP_PORTS.INPUT_BUFFERING_SDP_PORT.value)
sdp_message = SDPMessage(sdp_header, message.bytestring)
self._transceiver.send_sdp_message(sdp_message)
def send_port_trigger_message(connection, board_address):
""" Sends a port trigger message using a connection to (hopefully) open a\
port in a NAT and/or firewall to allow incoming packets to be received.
:param UDPConnection connection:
The UDP connection down which the trigger message should be sent
:param str board_address:
The IP address of the SpiNNaker board to which the message should be
sent
"""
# Set up the message so that no reply is expected and it is sent to an
# invalid port for SCAMP. The current version of SCAMP will reject
# this message, but then fail to send a response since the
# REPLY_NOT_EXPECTED flag is set (see scamp-3.c line 728 and 625-644)
trigger_message = SDPMessage(SDPHeader(
flags=SDPFlag.REPLY_NOT_EXPECTED, tag=0, destination_port=3,
destination_cpu=0, destination_chip_x=0, destination_chip_y=0))
update_sdp_header_for_udp_send(trigger_message.sdp_header, 0, 0)
connection.send_to(
trigger_message.bytestring, (board_address, SCP_SCAMP_PORT))
def get_data(self, transceiver, placement):
data = struct.pack("<I", self.SDP_PACKET_START_SENDING_COMMAND_ID)
print("sending to core {}:{}:{}".format(placement.x, placement.y,
placement.p))
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# create socket
connection = UDPConnection(local_host=None, local_port=self.PORT)
# send
transceiver.send_sdp_message(message=message)
# receive
finished = False
first = True
seq_num = 1
seq_nums = set()
while not finished:
try:
data = connection.receive(
timeout=self.TIMEOUT_PER_RECEIVE_IN_SECONDS)
first, seq_num, seq_nums, finished = \
self._process_data(
data, first, seq_num, seq_nums, finished, placement,
transceiver)
except SpinnmanTimeoutException:
if not finished:
finished = self._transmit_missing_seq_nums(
seq_nums, transceiver, placement)
self._check(seq_nums)
return self._output
def _determine_and_retransmit_missing_seq_nums(self, seq_nums, transceiver,
placement, lost_seq_nums):
""" Determine if there are any missing sequence numbers, and if so\
retransmits the missing sequence numbers back to the core for\
retransmission.
:param seq_nums: the sequence numbers already received
:param transceiver: spinnman instance
:param placement: placement instance
:return: whether all packets are transmitted
:rtype: bool
"""
# pylint: disable=too-many-locals
# locate missing sequence numbers from pile
missing_seq_nums = self._calculate_missing_seq_nums(seq_nums)
lost_seq_nums.append(len(missing_seq_nums))
# self._print_missing(missing_seq_nums)
if not missing_seq_nums:
return True
# figure n packets given the 2 formats
n_packets = 1
length_via_format2 = \
len(missing_seq_nums) - (self.DATA_PER_FULL_PACKET - 2)
if length_via_format2 > 0:
n_packets += int(
math.ceil(
float(length_via_format2) /
float(self.DATA_PER_FULL_PACKET - 1)))
# transmit missing sequence as a new SDP packet
first = True
seq_num_offset = 0
for _ in xrange(n_packets):
length_left_in_packet = self.DATA_PER_FULL_PACKET
offset = 0
# if first, add n packets to list
if first:
# get left over space / data size
size_of_data_left_to_transmit = min(
length_left_in_packet - 2,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 2) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag and n packets
_ONE_WORD.pack_into(
data, offset, self.SDP_PACKET_START_MISSING_SEQ_COMMAND_ID)
_ONE_WORD.pack_into(data, self.WORD_TO_BYTE_CONVERTER,
n_packets)
# update state
offset += 2 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 2
first = False
else: # just add data
# get left over space / data size
size_of_data_left_to_transmit = min(
self.DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 1) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag
_ONE_WORD.pack_into(data, offset,
self.SDP_PACKET_MISSING_SEQ_COMMAND_ID)
offset += 1 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 1
# fill data field
struct.pack_into(
"<{}I".format(size_of_data_left_to_transmit), data, offset,
*missing_seq_nums[seq_num_offset:seq_num_offset +
size_of_data_left_to_transmit])
seq_num_offset += length_left_in_packet
# build SDP message and send it to the core
transceiver.send_sdp_message(message=SDPMessage(
sdp_header=SDPHeader(destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data))
# sleep for ensuring core doesn't lose packets
time.sleep(self.TIME_OUT_FOR_SENDING_IN_SECONDS)
# self._print_packet_num_being_sent(packet_count, n_packets)
return False
def get_data(self, transceiver, placement, memory_address, length_in_bytes,
fixed_routes):
""" Gets data from a given core and memory address.
:param transceiver: spinnman instance
:param placement: placement object for where to get data from
:param memory_address: the address in SDRAM to start reading from
:param length_in_bytes: the length of data to read in bytes
:param fixed_routes: the fixed routes, used in the report of which\
chips were used by the speed up process
:return: byte array of the data
"""
start = float(time.time())
# if asked for no data, just return a empty byte array
if length_in_bytes == 0:
data = bytearray(0)
end = float(time.time())
self._provenance_data_items[placement, memory_address,
length_in_bytes].append(
(end - start, [0]))
return data
if (length_in_bytes <
self.THRESHOLD_WHERE_SDP_BETTER_THAN_DATA_EXTRACTOR_IN_BYTES):
data = transceiver.read_memory(placement.x, placement.y,
memory_address, length_in_bytes)
end = float(time.time())
self._provenance_data_items[placement, memory_address,
length_in_bytes].append(
(end - start, [0]))
return data
data = _THREE_WORDS.pack(self.SDP_PACKET_START_SENDING_COMMAND_ID,
memory_address, length_in_bytes)
# logger.debug("sending to core %d:%d:%d",
# placement.x, placement.y, placement.p)
# send
self._connection.send_sdp_message(
SDPMessage(sdp_header=SDPHeader(destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data))
# receive
self._output = bytearray(length_in_bytes)
self._view = memoryview(self._output)
self._max_seq_num = self.calculate_max_seq_num()
lost_seq_nums = self._receive_data(transceiver, placement)
end = float(time.time())
self._provenance_data_items[placement, memory_address,
length_in_bytes].append(
(end - start, lost_seq_nums))
# create report elements
if self._write_data_speed_up_report:
routers_been_in_use = self._determine_which_routers_were_used(
placement, fixed_routes, transceiver.get_machine_details())
self._write_routers_used_into_report(self._report_path,
routers_been_in_use,
placement)
return self._output
def receive_sdp_message(self, timeout=None):
data = self.receive(timeout)
return SDPMessage.from_bytestring(data, 2)
def _transmit_missing_seq_nums(self, seq_nums, transceiver, placement):
# locate missing seq nums from pile
missing_seq_nums = self._calculate_missing_seq_nums(seq_nums)
if len(missing_seq_nums) == 0:
return True
# figure n packets given the 2 formats
n_packets = 1
length_via_format2 = \
len(missing_seq_nums) - (self.DATA_PER_FULL_PACKET - 2)
if length_via_format2 > 0:
n_packets += int(
math.ceil(
float(length_via_format2) /
float(self.DATA_PER_FULL_PACKET - 1)))
# transmit missing seq as a new sdp packet
first = True
seq_num_offset = 0
for _packet_count in range(0, n_packets):
length_left_in_packet = self.DATA_PER_FULL_PACKET
offset = 0
data = None
size_of_data_left_to_transmit = None
# if first, add n packets to list
if first:
# get left over space / data size
size_of_data_left_to_transmit = min(
length_left_in_packet - 2,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 2) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag and n packets
struct.pack_into("<I", data, offset,
self.SDP_PACKET_START_MISSING_SEQ_COMMAND_ID)
struct.pack_into("<I", data, self.WORD_TO_BYTE_CONVERTER,
n_packets)
# update state
offset += 2 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 2
first = False
else: # just add data
# get left over space / data size
size_of_data_left_to_transmit = min(
self.DATA_PER_FULL_PACKET_WITH_SEQUENCE_NUM,
len(missing_seq_nums) - seq_num_offset)
# build data holder accordingly
data = bytearray((size_of_data_left_to_transmit + 1) *
self.WORD_TO_BYTE_CONVERTER)
# pack flag
struct.pack_into("<I", data, offset,
self.SDP_PACKET_MISSING_SEQ_COMMAND_ID)
offset += 1 * self.WORD_TO_BYTE_CONVERTER
length_left_in_packet -= 1
# fill data field
struct.pack_into(
"<{}I".format(size_of_data_left_to_transmit), data, offset,
*missing_seq_nums[seq_num_offset:seq_num_offset +
size_of_data_left_to_transmit])
seq_num_offset += length_left_in_packet
# build sdp message
message = SDPMessage(sdp_header=SDPHeader(
destination_chip_x=placement.x,
destination_chip_y=placement.y,
destination_cpu=placement.p,
destination_port=self.SDP_PACKET_PORT,
flags=SDPFlag.REPLY_NOT_EXPECTED),
data=data)
# debug
# self.print_out_missing_seq_packets_data(data)
# send message to core
transceiver.send_sdp_message(message=message)
# sleep for ensuring core doesn't lose packets
time.sleep(self.TIME_OUT_FOR_SENDING_IN_SECONDS)
# self._print_packet_num_being_sent(packet_count, n_packets)
return False
def receive_sdp_message(self, timeout=None):
data = self.receive(timeout)
return SDPMessage.from_bytestring(data, 2)
def _retrieve_and_store_data(self, packet):
""" Following a SpinnakerRequestReadData packet, the data stored\
during the simulation needs to be read by the host and stored in a\
data structure, following the specifications of buffering out\
technique
:param packet: SpinnakerRequestReadData packet received from the\
SpiNNaker system
:type packet:\
:py:class:`spinnman.messages.eieio.command_messages.spinnaker_request_read_data.SpinnakerRequestReadData`
:rtype: None
"""
x = packet.x
y = packet.y
p = packet.p
# check packet sequence number
pkt_seq = packet.sequence_no
last_pkt_seq = self._received_data.last_sequence_no_for_core(x, y, p)
next_pkt_seq = (last_pkt_seq + 1) % 256
if pkt_seq != next_pkt_seq:
# this sequence number is incorrect
# re-sent last HostDataRead packet sent
last_packet_sent = self._received_data.last_sent_packet_to_core(
x, y, p)
if last_packet_sent is not None:
self._transceiver.send_sdp_message(last_packet_sent)
else:
raise Exception(
"{}, {}, {}: Something somewhere went terribly wrong - "
"The packet sequence numbers have gone wrong "
"somewhere: the packet sent from the board "
"has incorrect sequence number, but the host "
"never sent one acknowledge".format(x, y, p))
return
# read data from memory, store it and create data for return ACK packet
n_requests = packet.n_requests
new_channel = list()
new_region_id = list()
new_space_read = list()
new_n_requests = 0
for i in xrange(n_requests):
length = packet.space_to_be_read(i)
if length > 0:
new_n_requests += 1
start_address = packet.start_address(i)
region_id = packet.region_id(i)
channel = packet.channel(i)
data = self._transceiver.read_memory(x, y, start_address,
length)
self._received_data.store_data_in_region_buffer(
x, y, p, region_id, data)
new_channel.append(channel)
new_region_id.append(region_id)
new_space_read.append(length)
# create return acknowledge packet with data stored
ack_packet = HostDataRead(new_n_requests, pkt_seq, new_channel,
new_region_id, new_space_read)
ack_packet_data = ack_packet.bytestring
# create SDP header and message
return_message_header = SDPHeader(
destination_port=SDP_PORTS.OUTPUT_BUFFERING_SDP_PORT.value,
destination_cpu=p,
destination_chip_x=x,
destination_chip_y=y,
flags=SDPFlag.REPLY_NOT_EXPECTED)
return_message = SDPMessage(return_message_header, ack_packet_data)
# storage of last packet received
self._received_data.store_last_received_packet_from_core(
x, y, p, packet)
self._received_data.update_sequence_no_for_core(x, y, p, pkt_seq)
# store last sent message and send to the appropriate core
self._received_data.store_last_sent_packet_to_core(
x, y, p, return_message)
self._transceiver.send_sdp_message(return_message)
