def close(self):
"""Shutdown connections to FPGA if applicable"""
# Function does nothing if FPGA configuration not found in config file
if not self.config_found:
return
# Close the UDP socket if it is open
if self.udp_socket is not None:
# Send termination signal to the board
self.terminate_client()
# Close the udp socket and set it to None
logger.info("<%s> UDP Connection closed",
fpga_config.get(self.fpga_name, "ip"))
self.udp_socket.close()
self.udp_socket = None
# Reset the udp communication buffers
self.send_buffer = np.zeros(self.input_dimensions +
self.output_dimensions + 1)
self.recv_buffer = np.zeros(self.output_dimensions + 1)
# Close the SSH connection
logger.info("<%s> SSH connection closed",
fpga_config.get(self.fpga_name, "ip"))
self.ssh_client.close()
def connect_thread_func(self):
"""Start SSH in a separate thread to monitor status"""
# # Get the IP of the remote device from the fpga_config file
remote_ip = fpga_config.get(self.fpga_name, "ip")
# # Get the SSH options from the fpga_config file
ssh_user = fpga_config.get(self.fpga_name, "ssh_user")
self.connect_ssh_client(ssh_user, remote_ip)
# Invoke a shell in the ssh client
ssh_channel = self.ssh_client.invoke_shell()
# If board configuration specifies using sudo to run scripts
# - Assume all non-root users will require sudo to run the scripts
# - Note: Also assumes that the fpga has been configured to allow
# the ssh user to run sudo commands WITHOUT needing a password
# (see specific fpga hardware docs for details)
if ssh_user != "root":
print(f"<{remote_ip}> Script to be run with sudo. Sudoing.",
flush=True)
ssh_channel.send("sudo su\n")
# Send required ssh string
print(
f"<{fpga_config.get(self.fpga_name, 'ip')}> Sending cmd to fpga board: \n"
f"{self.ssh_string}",
flush=True,
)
ssh_channel.send(self.ssh_string)
# Variable for remote error handling
got_error = 0
error_strs = []
# Get and process the information being returned over the ssh
# connection
while True:
data = ssh_channel.recv(256)
if not data:
# If no data is received, the client has been closed, so close
# the channel, and break out of the while loop
ssh_channel.close()
break
self.process_ssh_output(data)
info_str_list = self.ssh_info_str.split("\n")
for info_str in info_str_list[:-1]:
got_error, error_strs = self.check_ssh_str(
info_str, error_strs, got_error, remote_ip)
self.ssh_info_str = info_str_list[-1]
# The traceback usually contains 3 lines, so collect the first
# three lines then display it.
if got_error == 2:
ssh_channel.close()
raise RuntimeError(
f"Received the following error on the remote side <{remote_ip}>:\n"
+ "\n".join(error_strs))
def ssh_string(self):
"""Command sent to FPGA device to begin execution
Generate the string to be sent over the ssh connection to run the
remote side ssh script (with appropriate arguments)
"""
ssh_str = ""
if self.config_found:
ssh_str = ("python " +
fpga_config.get(self.fpga_name, "id_script") +
f" --host_ip=\"{fpga_config.get('host', 'ip')}\"" +
f" --tcp_port={self.tcp_port}" + "\n")
return ssh_str
def __init__(self, fpga_name, max_attempts=5, timeout=5):
self.config_found = fpga_config.has_section(fpga_name)
self.fpga_name = fpga_name
self.max_attempts = max_attempts
self.timeout = timeout
# Make SSHClient object
self.ssh_client = paramiko.SSHClient()
self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh_info_str = ""
self.ssh_lock = False
# Check if the desired FPGA name is defined in the configuration file
if self.config_found:
# Handle the tcp port selection: Use the config specified port.
# If none is provided (i.e., the specified port number is 0),
# choose a random tcp port number between 20000 and 65535.
# We will use the udp port number from the config but use tcp.
self.tcp_port = int(fpga_config.get(fpga_name, "udp_port"))
if self.tcp_port == 0:
self.tcp_port = int(np.random.uniform(low=20000, high=65535))
# Make the TCP socket for receiving Device ID.
self.tcp_init = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.tcp_init.bind((fpga_config.get("host", "ip"), self.tcp_port))
self.tcp_init.settimeout(self.timeout)
self.tcp_init.listen(1) # Ready to accept a connection
self.tcp_recv = None # Placeholder until socket is connected
else:
# FPGA name not found, unable to retrieve ID
print(
"ERROR: Specified FPGA configuration '" + fpga_name +
"' not found.",
flush=True,
)
sys.exit()
def reset(self):
"""Reconnect to FPGA if applicable"""
# Function does nothing if FPGA configuration not found in config file
if not self.config_found:
return
# Otherwise, close and reopen the SSH connection to the board
# Closing the SSH connection will terminate the board-side script
logger.info("<%s> Resetting SSH connection:",
fpga_config.get(self.fpga_name, "ip"))
# Close and reopen ssh connections
self.close()
self.connect()
def connect_ssh_client(self, ssh_user, remote_ip):
"""Helper function to parse config and setup ssh client"""
# Get the SSH options from the fpga_config file
ssh_port = fpga_config.get(self.fpga_name, "ssh_port")
if fpga_config.has_option(self.fpga_name, "ssh_pwd"):
ssh_pwd = fpga_config.get(self.fpga_name, "ssh_pwd")
else:
ssh_pwd = None
if fpga_config.has_option(self.fpga_name, "ssh_key"):
ssh_key = os.path.expanduser(
fpga_config.get(self.fpga_name, "ssh_key"))
else:
ssh_key = None
# Connect to remote location over ssh
if ssh_key is not None:
# If an ssh key is provided, just use it
self.ssh_client.connect(remote_ip,
port=ssh_port,
username=ssh_user,
key_filename=ssh_key)
elif ssh_pwd is not None:
# If an ssh password is provided, just use it
self.ssh_client.connect(remote_ip,
port=ssh_port,
username=ssh_user,
password=ssh_pwd)
else:
# If no password or key is specified, just use the default connect
# (paramiko will then try to connect using the id_rsa file in the
# ~/.ssh/ folder)
self.ssh_client.connect(remote_ip,
port=ssh_port,
username=ssh_user)
def ssh_string(self):
"""Command sent to FPGA device to begin execution
Generate the string to be sent over the ssh connection to run the
remote side ssh script (with appropriate arguments)
"""
ssh_str = ""
if self.config_found:
ssh_str = (
"python " + fpga_config.get(self.fpga_name, "remote_script") +
f" --host_ip='{fpga_config.get('host', 'ip')}'" +
f" --remote_ip='{fpga_config.get(self.fpga_name, 'ip')}'" +
f" --udp_port={self.udp_port}" +
f" --arg_data_file='{fpga_config.get(self.fpga_name, 'remote_tmp')}"
+ f"/{self.arg_data_file}'" + "\n")
return ssh_str
def connect(self): # noqa: C901
"""Connect to FPGA via SSH if applicable"""
# Function does nothing if FPGA configuration not found in config file
if not self.config_found:
return
logger.info("<%s> Open SSH connection",
fpga_config.get(self.fpga_name, "ip"))
# Start a new thread to open the ssh connection. Use a thread to
# handle the opening of the connection because it can lag for certain
# devices, and we don't want it to impact the rest of the build process.
connect_thread = threading.Thread(target=self.connect_thread_func,
args=())
connect_thread.start()
logger.info("<%s> Open UDP connection",
fpga_config.get(self.fpga_name, "ip"))
# Create a UDP socket to communicate with the board
self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.udp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.udp_socket.bind((fpga_config.get("host", "ip"), self.udp_port))
# # Set the socket timeout to the connection timeout and wait for the
# # board to connect to the PC.
# # Note that this is the "correct" way to wait for an incoming connection
# # request. However, because the error detection mechanism in the SSH
# # thread is done by closing the UDP socket, a polling approach is used
# # (see below).
# # TODO: Re-examine this during the SSH refactoring.
# self.udp_socket.settimeout(self.connect_timeout)
# while True:
# try:
# self.udp_socket.recv_into(self.recv_buffer.data)
# # Connection packet has a t of 0
# if self.recv_buffer[0] <= 0.0:
# break
# except socket.timeout:
# self.close()
# raise RuntimeError("Did not receive connection from board within "
# + "specified timeout (%fs)." % self.connect_timeout)
# if self.recv_buffer[0] < 0.0:
# self.close()
# raise RuntimeError("Simulation terminated by FPGA board.")
# Connection with the board established. Set the socket timeout to
# recv_timeout.
self.udp_socket.settimeout(self.recv_timeout)
# Wait for the connection packet from the board. Note that the
# implementation above (setting the socket timeout to ``connect_timeout``)
# is the "proper" implementation. Below is a workaround to allow the SSH
# connection thread to terminate the connection process (by closing the
# udp socket, so an exception is thrown).
max_attempts = int(self.connect_timeout / self.recv_timeout)
for conn_attempts in range(max_attempts):
try:
self.udp_socket.recv_into(self.recv_buffer)
if self.recv_buffer[0] <= 0.0:
# Received a connection packet (t == 0) from the board, or received
# a "terminate client" packet (t < 0) from the board, so break out
# of the connection waiting loop
break
except socket.timeout:
pass
except AttributeError:
sys.exit(1)
if conn_attempts >= (max_attempts - 1):
# Number of connection attempts exceeds maximum number of attempts.
# I.e., no connection has been received within the timeout limit.
self.close()
raise RuntimeError(f"Did not receive connection from board within "
f"specified timeout ({self.connect_timeout}s).")
if self.recv_buffer[0] < 0.0:
# Received a "terminate client" packet from the board, terminate the
# Nengo simulation.
reason = ""
if self.recv_buffer[0] <= -20:
reason = "Unable to load FPGA driver! "
elif self.recv_buffer[0] <= -10:
reason = "Unable to acquire FPGA resource lock! "
self.close()
raise RuntimeError(reason + "Simulation terminated by FPGA board.")
def connect_thread_func(self):
"""Start SSH in a separate thread if applicable"""
# Function does nothing if FPGA configuration not found in config file
if not self.config_found:
return
# Get the IP of the remote device from the fpga_config file
remote_ip = fpga_config.get(self.fpga_name, "ip")
# Get the SSH options from the fpga_config file
ssh_user = fpga_config.get(self.fpga_name, "ssh_user")
self.connect_ssh_client(ssh_user, remote_ip)
# Send argument file over
remote_data_filepath = (
f"{fpga_config.get(self.fpga_name, 'remote_tmp')}/{self.arg_data_file}"
)
if os.path.exists(self.local_data_filepath):
logger.info(
"<%s> Sending argument data (%s) to fpga board",
fpga_config.get(self.fpga_name, "ip"),
self.arg_data_file,
)
# Send the argument data over to the fpga board
# Create sftp connection
sftp_client = self.ssh_client.open_sftp()
sftp_client.put(self.local_data_filepath, remote_data_filepath)
# Close sftp connection and release ssh connection lock
sftp_client.close()
# Invoke a shell in the ssh client
ssh_channel = self.ssh_client.invoke_shell()
# Wait for the SSH shell to initialize
time.sleep(0.1)
# If board configuration specifies using sudo to run scripts
# - Assume all non-root users will require sudo to run the scripts
# - Note: Also assumes that the fpga has been configured to allow
# the ssh user to run sudo commands WITHOUT needing a password
# (see specific fpga hardware docs for details)
if ssh_user != "root":
logger.info("<%s> Script to be run with sudo. Sudoing.", remote_ip)
ssh_channel.send("sudo su\n")
# Send required ssh string
logger.info(
"<%s> Sending cmd to fpga board: \n%s",
fpga_config.get(self.fpga_name, "ip"),
self.ssh_string,
)
ssh_channel.send(self.ssh_string)
# Variable for remote error handling
got_error = 0
error_strs = []
# Get and process the information being returned over the ssh
# connection
while True:
data = ssh_channel.recv(256)
if not data:
# If no data is received, the client has been closed, so close
# the channel, and break out of the while loop
ssh_channel.close()
break
self.process_ssh_output(data)
info_str_list = self.ssh_info_str.split("\n")
for info_str in info_str_list[:-1]:
got_error, error_strs = self.check_ssh_str(
info_str, error_strs, got_error, remote_ip)
self.ssh_info_str = info_str_list[-1]
# The traceback usually contains 3 lines, so collect the first
# three lines then display it.
if got_error == 2:
# Close the UDP and SSH connections so that the main simulation thread
# terminates
self.close()
raise RuntimeError(
f"Received the following error on the remote side <{remote_ip}>:\n"
+ "\n".join(error_strs))
logger.info("Terminating SSH thread")
def __init__(
self,
fpga_name,
n_neurons,
dimensions,
learning_rate,
function=nengo.Default,
transform=nengo.Default,
eval_points=nengo.Default,
socket_args=None,
feedback=None,
label=None,
seed=None,
add_to_container=None,
):
# Flags for determining whether or not the FPGA board is being used
self.config_found = fpga_config.has_section(fpga_name)
self.fpga_found = True # TODO: Ping board to determine?
self.using_fpga_sim = False
# Make SSHClient object
self.ssh_client = paramiko.SSHClient()
self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh_info_str = ""
self.ssh_lock = False
# Save ssh details
self.fpga_name = fpga_name
self.arg_data_path = os.curdir
self.arg_data_file = ""
# Process dimensions, function, transform arguments
self.input_dimensions = dimensions
self.get_output_dim(function, dimensions)
# Process feedback connection
if nengo.utils.numpy.is_array_like(feedback):
self.rec_transform = feedback
elif feedback is not None:
raise nengo.exceptions.ValidationError(
"Must be scalar or array-like", "feedback", self)
# Neuron type string map
self.neuron_str_map = {
nengo.neurons.RectifiedLinear: "RectifiedLinear",
nengo.neurons.SpikingRectifiedLinear: "SpikingRectifiedLinear",
}
self.default_neuron_type = nengo.neurons.RectifiedLinear()
# Store the various parameters needed to initialize the remote network
self.seed = seed
self.learning_rate = learning_rate
# Call the superconstructor
super().__init__(label, seed, add_to_container)
# Socket attributes
self.udp_socket = None
self.send_buffer = None
self.recv_buffer = None
if socket_args is None:
socket_args = {}
self.connect_timeout = socket_args.get("connect_timeout", 30)
self.recv_timeout = socket_args.get("recv_timeout", 0.1)
# Check if the desired FPGA name is defined in the configuration file
if self.config_found:
# Handle the udp port selection: Use the config specified port.
# If none is provided (i.e., the specified port number is 0),
# choose a random udp port number between 20000 and 65535.
self.udp_port = int(fpga_config.get(fpga_name, "udp_port"))
if self.udp_port == 0:
self.udp_port = int(np.random.uniform(low=20000, high=65535))
self.send_addr = (fpga_config.get(fpga_name, "ip"), self.udp_port)
else:
# FPGA name not found, throw a warning.
logger.warning("Specified FPGA configuration '%s' not found.",
fpga_name)
print(
f"WARNING: Specified FPGA configuration '{fpga_name}' not found."
)
# Make nengo model. Here, a dummy ensemble is created. It will be
# replaced with a udp_socket in the builder function (see below).
with self:
self.input = nengo.Node(size_in=self.input_dimensions,
label="input")
self.error = nengo.Node(size_in=self.output_dimensions,
label="error")
self.output = nengo.Node(size_in=self.output_dimensions,
label="output")
self.ensemble = nengo.Ensemble(
n_neurons,
self.input_dimensions,
neuron_type=nengo.neurons.RectifiedLinear(),
eval_points=eval_points,
label="Dummy Ensemble",
)
nengo.Connection(self.input, self.ensemble, synapse=None)
self.connection = nengo.Connection(
self.ensemble,
self.output,
function=function,
transform=transform,
eval_points=eval_points,
learning_rule_type=nengo.PES(learning_rate),
)
nengo.Connection(self.error,
self.connection.learning_rule,
synapse=None)
if feedback is not None:
self.feedback = nengo.Connection(
self.ensemble,
self.ensemble,
synapse=nengo.Lowpass(0.1),
transform=self.rec_transform,
)
else:
self.feedback = None
# Make the object lists immutable so that no extra objects can be added
# to this network.
for k, v in self.objects.items():
self.objects[k] = tuple(v)