class SourceCom:
def __init__(self, sending_topics, parameters_topic, port, worker_exec, verbose='||',
ssh_local_server_id='None', ssh_remote_server_id='None', outputs=None):
self.sending_topics = sending_topics
self.parameters_topic = parameters_topic
self.pull_data_port = port
self.heartbeat_port = str(int(self.pull_data_port) + 1)
self.worker_exec = worker_exec
self.index = 0
self.time = int(1000000 * time.perf_counter())
self.previous_time = self.time
self.verbose, self.relic = self.define_verbosity_and_relic(verbose)
self.all_loops_running = True
self.ssh_com = SSHCom(self.worker_exec, ssh_local_server_id, ssh_remote_server_id)
self.outputs = outputs
self.port_pub = ct.DATA_FORWARDER_SUBMIT_PORT
self.context = None
self.socket_pub_data = None
self.socket_pull_data = None
self.stream_pull_data = None
self.socket_push_heartbeat = None
self.average_sending_time = 0
# If self.verbose is a string it is the file name to log things in. If it is an int it is the level of the verbosity
self.logger = None
if self.verbose != 0:
try:
self.verbose = int(self.verbose)
except:
log_file_name = gu.add_timestamp_to_filename(self.verbose, datetime.now())
self.logger = gu.setup_logger('Source', log_file_name)
self.logger.info('Index of data packet : Computer Time Data Out')
self.verbose = False
def connect_sockets(self):
"""
Start the required sockets to communicate with the link forwarder and the source_com processes
:return: Nothing
"""
if self.verbose:
print('Starting Source Node with PID = {}'.format(os.getpid()))
self.context = zmq.Context()
# Socket for pulling the data from the worker_exec
self.socket_pull_data = Socket(self.context, zmq.PULL)
self.socket_pull_data.setsockopt(zmq.LINGER, 0)
self.socket_pull_data.set_hwm(1)
self.socket_pull_data.connect(r"tcp://127.0.0.1:{}".format(self.pull_data_port))
self.stream_pull_data = zmqstream.ZMQStream(self.socket_pull_data)
self.stream_pull_data.on_recv(self.on_receive_data_from_worker)
# Socket for publishing the data to the data forwarder
self.socket_pub_data = Socket(self.context, zmq.PUB)
self.socket_pub_data.setsockopt(zmq.LINGER, 0)
self.socket_pub_data.set_hwm(len(self.sending_topics))
self.socket_pub_data.connect(r"tcp://127.0.0.1:{}".format(self.port_pub))
# Socket for publishing the heartbeat to the worker_exec
self.socket_push_heartbeat = self.context.socket(zmq.PUSH)
self.socket_push_heartbeat.setsockopt(zmq.LINGER, 0)
self.socket_push_heartbeat.bind(r'tcp://*:{}'.format(self.heartbeat_port))
self.socket_push_heartbeat.set_hwm(1)
def define_verbosity_and_relic(self, verbosity_string):
"""
Splits the string that comes from the Node as verbosity_string into the string (or int) for the logging/printing
(self.verbose) and the string that carries the path where the relic is to be saved. The self.relic is then
passed to the worker process
:param verbosity_string: The string with syntax verbosity||relic
:return: (int)str vebrose, str relic
"""
if verbosity_string != '':
verbosity, relic = verbosity_string.split('||')
if relic == '':
relic = '_'
if verbosity == '':
return 0, relic
else:
return verbosity, relic
else:
return 0, ''
def on_receive_data_from_worker(self, msg):
"""
The callback that runs every time link is received from the worker_exec process. It takes the link and passes it
onto the link forwarder
:param msg: The link packet (carrying the actual link (np array))
:return:
"""
# A specific worker with multiple outputs should send from its infinite loop a message with multiple parts
# (using multiple send_array(data, flags=zmq.SNDMORE) commands). For an example see how the transform_worker
# sends data to the com from its data_callback function
# TODO The bellow will not work for multiple outputs. I have to find out how many times a callback is called
# when data are send with SNDMORE flag !!!
ignoring_outputs = [False] * len(self.outputs)
new_message_data = []
if len(self.outputs) > 1:
for i in range(len(self.outputs)):
array_data = Socket.reconstruct_array_from_bytes_message(msg[i])
new_message_data.append(array_data)
if type(array_data[0]) == np.str_:
if array_data[0] == ct.IGNORE:
ignoring_outputs[i] = True
else:
array_data = Socket.reconstruct_array_from_bytes_message(msg)
new_message_data.append(array_data)
if type(array_data[0]) == np.str_:
if array_data[0] == ct.IGNORE:
ignoring_outputs[0] = True
self.time = int(1000000 * time.perf_counter())
self.index = self.index + 1
# Publish the results. Each array in the list of arrays is published to its own sending topic
# (matched by order)
for i, st in enumerate(self.sending_topics):
for k, output in enumerate(self.outputs):
if output.replace(' ', '_') in st.split('##')[0]:
break
if ignoring_outputs[k] is False:
self.socket_pub_data.send("{}".format(st).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send("{}".format(self.index).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send("{}".format(self.time).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send_array(new_message_data[k], copy=False)
# This delay is critical to get single output to multiple inputs to work!
gu.accurate_delay(ct.DELAY_BETWEEN_SENDING_DATA_TO_NEXT_NODE_MILLISECONDS)
if self.verbose:
dt = self.time - self.previous_time
if self.index > 3:
self.average_sending_time = self.average_sending_time * (self.index - 1) / self.index + dt / self.index
print('----------')
print("Source with topic {} sending packet with data_index {} at time {}".format(self.sending_topics[i],
self.index, self.time))
print('Time Diff between packages = {}. Average package sending time = {} ms'.format(dt/1000, self.average_sending_time / 1000))
if self.logger:
self.logger.info('{} : {}'.format(self.index, datetime.now()))
self.previous_time = self.time
def heartbeat_loop(self):
"""
Sending every ct.HEARTBEAT_RATE a 'PULSE' to the worker_exec so that it stays alive
:return: Nothing
"""
while self.all_loops_running:
self.socket_push_heartbeat.send_string('PULSE')
time.sleep(ct.HEARTBEAT_RATE)
def start_heartbeat_thread(self):
"""
Starts the daemon thread that runs the self.heartbeat loop
:return: Nothing
"""
heartbeat_thread = threading.Thread(target=self.heartbeat_loop, daemon=True)
heartbeat_thread.start()
def start_worker_process(self):
"""
Starts the worker_exec process and then sends the parameters as are currently on the node to the process
The pull_data_port of the worker_exec needs to be the push_data_port of the com (obviously).
The way the arguments are structured is defined by the way they are read by the process. For that see
general_utilities.parse_arguments_to_worker
:return: Nothing
"""
if 'python' in self.worker_exec or '.py' not in self.worker_exec:
arguments_list = [self.worker_exec]
else:
arguments_list = ['python']
arguments_list.append(self.worker_exec)
arguments_list.append(str(self.pull_data_port))
arguments_list.append(str(self.parameters_topic))
arguments_list.append(str(0))
arguments_list.append(str(len(self.sending_topics)))
arguments_list.append(self.relic)
arguments_list = self.ssh_com.add_local_server_info_to_arguments(arguments_list)
worker_pid = self.ssh_com.start_process(arguments_list)
self.ssh_com.connect_socket_to_remote(self.socket_pull_data,
r"tcp://127.0.0.1:{}".format(self.pull_data_port))
def start_ioloop(self):
"""
Starts the ioloop of the zmqstream
:return: Nothing
"""
ioloop.IOLoop.instance().start()
def on_kill(self, signal, frame):
"""
The function that is called when the parent process sends a SIGBREAK (windows) or SIGTERM (linux) signal.
It needs signal and frame as parameters
:param signal: The signal received
:param frame: I haven't got a clue
:return: Nothing
"""
try:
self.all_loops_running = False
self.stream_pull_data.close(linger=0)
self.socket_pull_data.close()
self.socket_pub_data.close()
self.socket_push_heartbeat.close()
except Exception as e:
print('Trying to kill Source com {} failed with error: {}'.format(self.sending_topics[0], e))
finally:
self.context.term()
class SinkCom:
def __init__(self,
receiving_topics,
parameters_topic,
push_port,
worker_exec,
verbose=True,
ssh_local_server_id='None',
ssh_remote_server_id='None'):
self.receiving_topics = receiving_topics
self.parameters_topic = parameters_topic
self.push_data_port = push_port
self.pull_data_port = str(int(self.push_data_port) + 1)
self.push_heartbeat_port = str(int(self.push_data_port) + 2)
self.worker_exec = worker_exec
self.verbose = verbose
self.verbose, self.relic = self.define_verbosity_and_relic(verbose)
self.all_loops_running = True
self.ssh_com = SSHCom(self.worker_exec, ssh_local_server_id,
ssh_remote_server_id)
self.port_sub_data = ct.DATA_FORWARDER_PUBLISH_PORT
self.port_pub_parameters = ct.PARAMETERS_FORWARDER_SUBMIT_PORT
self.poller = zmq.Poller()
self.context = None
self.socket_sub_data = None
self.stream_sub = None
self.socket_push_data = None
self.socket_pull_data = None
self.socket_push_heartbeat = None
self.index = 0
# If self.verbose is a string it is the file name to log things in. If it is an int it is the level of the verbosity
self.logger = None
if self.verbose != 0:
try:
self.verbose = int(self.verbose)
except:
log_file_name = gu.add_timestamp_to_filename(
self.verbose, datetime.now())
self.logger = gu.setup_logger('Sink', log_file_name)
self.logger.info(
'Index of data packet given : Index of data packet received: Topic : Computer Time'
)
self.verbose = False
atexit.register(self.on_kill, None, None)
signal.signal(signal.SIGTERM, self.on_kill)
def connect_sockets(self):
"""
Start the required sockets to communicate with the link forwarder and the worker_com processes
:return: Nothing
"""
if self.verbose:
print('Starting Sink Node with PID = {}'.format(os.getpid()))
self.context = zmq.Context()
# Socket for subscribing to data from nodes connected to the input
self.socket_sub_data = Socket(self.context, zmq.SUB)
self.socket_sub_data.setsockopt(zmq.LINGER, 0)
self.socket_sub_data.set_hwm(len(self.receiving_topics))
self.socket_sub_data.connect("tcp://127.0.0.1:{}".format(
self.port_sub_data))
for rt in self.receiving_topics:
self.socket_sub_data.setsockopt(zmq.SUBSCRIBE, rt.encode('ascii'))
self.poller.register(self.socket_sub_data, zmq.POLLIN)
# Socket for pushing the data to the worker_exec
self.socket_push_data = Socket(self.context, zmq.PUSH)
self.socket_push_data.setsockopt(zmq.LINGER, 0)
self.socket_push_data.set_hwm(1)
self.socket_push_data.bind(r"tcp://*:{}".format(self.push_data_port))
# Socket for pulling the end of worker function signal from the worker_exec
self.socket_pull_data = Socket(self.context, zmq.PULL)
self.socket_pull_data.setsockopt(zmq.LINGER, 0)
self.socket_pull_data.set_hwm(1)
self.socket_pull_data.connect(r"tcp://127.0.0.1:{}".format(
self.pull_data_port))
self.poller.register(self.socket_pull_data, zmq.POLLIN)
# Socket for pushing the heartbeat to the worker_exec
self.socket_push_heartbeat = self.context.socket(zmq.PUSH)
self.socket_push_heartbeat.setsockopt(zmq.LINGER, 0)
self.socket_push_heartbeat.bind(r'tcp://*:{}'.format(
self.push_heartbeat_port))
self.socket_push_heartbeat.set_hwm(1)
def define_verbosity_and_relic(self, verbosity_string):
"""
Splits the string that comes from the Node as verbosity_string into the string (or int) for the logging/printing
(self.verbose) and the string that carries the path where the relic is to be saved. The self.relic is then
passed to the worker process
:param verbosity_string: The string with syntax verbosity||relic
:return: (int)str vebrose, str relic
"""
if verbosity_string != '':
verbosity, relic = verbosity_string.split('||')
if relic == '':
relic = '_'
if verbosity == '':
return 0, relic
else:
return verbosity, relic
else:
return 0, ''
def heartbeat_loop(self):
"""
The loop that send a 'PULSE' heartbeat to the worker_exec process to keep it alive (every ct.HEARTBEAT_RATE seconds)
:return: Nothing
"""
while self.all_loops_running:
self.socket_push_heartbeat.send_string('PULSE')
time.sleep(ct.HEARTBEAT_RATE)
def start_heartbeat_thread(self):
"""
The daemon thread that runs the infinite heartbeat_loop
:return: Noting
"""
heartbeat_thread = threading.Thread(target=self.heartbeat_loop,
daemon=True)
heartbeat_thread.start()
def start_worker(self):
"""
Starts the worker_exec process and then sends the parameters as are currently on the node to the process
The pull_data_port of the worker_exec needs to be the push_data_port of the com (obviously).
The way the arguments are structured is defined by the way they are read by the process. For that see
general_utilities.parse_arguments_to_worker
:return: Nothing
"""
if 'python' in self.worker_exec or '.py' not in self.worker_exec:
arguments_list = [self.worker_exec]
else:
arguments_list = ['python']
arguments_list.append(self.worker_exec)
arguments_list.append(str(self.push_data_port))
arguments_list.append(str(self.parameters_topic))
arguments_list.append(str(len(self.receiving_topics)))
for i in range(len(self.receiving_topics)):
arguments_list.append(self.receiving_topics[i])
arguments_list.append(str(0))
arguments_list.append(str(self.relic))
arguments_list = self.ssh_com.add_local_server_info_to_arguments(
arguments_list)
worker_pid = self.ssh_com.start_process(arguments_list)
self.ssh_com.connect_socket_to_remote(
self.socket_pull_data,
r"tcp://127.0.0.1:{}".format(self.pull_data_port))
def get_sub_data(self):
"""
Gets the link from the forwarder. It assumes that each message has four parts:
The topic
The data_index, an int that increases by one for every message the previous node sends
The data_time, the time.perf_counter() result at the time the previous node send its message
The messagedata, the array of link
:return: Nothing
"""
prev_topic = self.socket_sub_data.recv()
prev_data_index = self.socket_sub_data.recv()
prev_data_time = self.socket_sub_data.recv()
prev_messagedata = self.socket_sub_data.recv_array()
# The following while ensures that the sink works only on the the latest available
# message from the previous node. If the sink is too slow compared to the input node
# this while throws all past messages away.
while prev_topic:
topic = prev_topic
data_index = prev_data_index
data_time = prev_data_time
messagedata = prev_messagedata
try:
prev_topic = self.socket_sub_data.recv(zmq.NOBLOCK)
prev_data_index = self.socket_sub_data.recv(zmq.NOBLOCK)
prev_data_time = self.socket_sub_data.recv(zmq.NOBLOCK)
prev_messagedata = self.socket_sub_data.recv_array(zmq.NOBLOCK)
except:
prev_topic = None
pass
return topic, data_index, data_time, messagedata
def start_ioloop(self):
"""
Start the io loop for the sink node. It reads the link from the previous node's _com process,
pushes it to the worker_com process,
waits for the results,
grabs the resulting link from the worker_com process and
publishes the transformed link to the link forwarder with this nodes' topic
:return: Nothing
"""
while self.all_loops_running:
t1 = time.perf_counter()
try:
# The timeout=1 means things coming in faster than 1000Hz will be lost but if timeout is set to 0 then
# the CPU utilization goes to around 10% which quickly kills the CPU (if there are 2 or 3 Sinks in the
# pipeline)
sockets_in = dict(self.poller.poll(timeout=1))
while not sockets_in:
sockets_in = dict(self.poller.poll(timeout=1))
if self.socket_sub_data in sockets_in and sockets_in[
self.socket_sub_data] == zmq.POLLIN:
topic, data_index, data_time, messagedata = self.get_sub_data(
)
sockets_in = dict(self.poller.poll(timeout=1))
if self.verbose:
print(
"oooo Sink from {}, data_index {} at time {} s oooo"
.format(topic, data_index, data_time))
# Send link to be transformed to the worker_exec
self.socket_push_data.send(topic, flags=zmq.SNDMORE)
self.socket_push_data.send_array(messagedata, copy=False)
t2 = time.perf_counter()
# Get the end of worker function link (wait for the socket_pull_data to get some link from the worker_exec)
sockets_in = dict(self.poller.poll(timeout=None))
self.socket_pull_data.recv()
t3 = time.perf_counter()
if self.verbose:
print(
"---Time to Transport link from previous com to worker_exec = {} ms"
.format((t2 - t1) * 1000))
print("---Time to to finish with the worker_exec = {} ms".
format((t3 - t2) * 1000))
print('=============================')
if self.logger:
self.logger.info('{} : {} : {} : {}'.format(
self.index, data_index, topic, datetime.now()))
self.index += 1
except:
pass
def on_kill(self, signal, frame):
try:
self.all_loops_running = False
self.poller.unregister(socket=self.socket_sub_data)
self.poller.unregister(socket=self.socket_pull_data)
self.socket_sub_data.close()
self.socket_push_data.close()
self.socket_pull_data.close()
self.socket_push_heartbeat.close()
except Exception as e:
print('Trying to kill Sink com {} failed with error: {}'.format(
self.worker_exec, e))
finally:
self.context.term()
class Node:
def __init__(self, name, parent):
self.id = None
self.name = name
self.parent = parent
self.operation = None
self.node_index = None
self.process = None
self.topics_out = []
self.topics_in = []
self.links_list = []
self.starting_port = None
self.num_of_inputs = 0
self.num_of_outputs = 0
self.coordinates = [100, 100]
self.node_parameters = None
self.node_parameters_combos_items = []
self.parameter_inputs_ids = {}
self.com_verbosity = ''
self.relic_verbosity = ''
self.verbose = ''
self.context = None
self.socket_pub_parameters = None
self.socket_sub_proof_of_life = None
self.theme_id = None
self.extra_window_id = None
self.operations_list = op.operations_list
self.get_corresponding_operation()
self.get_node_index()
self.assign_default_parameters()
self.get_numbers_of_inputs_and_outputs()
#self.generate_default_topics()
self.ssh_server_id_and_names = None
self.get_ssh_server_names_and_ids()
self.ssh_local_server = self.ssh_server_id_and_names[0]
self.ssh_remote_server = self.ssh_server_id_and_names[0]
self.worker_executable = self.operation.worker_exec
def get_attribute_order(self, type):
number_of_static_attrs = 0
for at in self.operation.attribute_types:
if 'Static' in at:
number_of_static_attrs += 1
number_of_input_attrs = 0
number_of_output_attrs = 0
order = {}
for at, attr_order_num in zip(self.operation.attribute_types,
range(len(self.operation.attributes))):
if type == 'Input' and type in at:
order[self.operation.
attributes[attr_order_num]] = number_of_input_attrs
number_of_input_attrs += 1
if type == 'Output' and type in at:
order[self.operation.
attributes[attr_order_num]] = number_of_output_attrs
number_of_output_attrs += 1
return order
def initialise_parameters_socket(self):
if self.context is None:
self.context = zmq.Context()
self.socket_pub_parameters = Socket(self.context, zmq.PUB)
self.socket_pub_parameters.setsockopt(zmq.LINGER, 0)
self.socket_pub_parameters.connect(r"tcp://127.0.0.1:{}".format(
ct.PARAMETERS_FORWARDER_SUBMIT_PORT))
def initialise_proof_of_life_socket(self):
if self.context is None:
self.context = zmq.Context()
self.socket_sub_proof_of_life = self.context.socket(zmq.SUB)
self.socket_sub_proof_of_life.setsockopt(zmq.LINGER, 0)
self.socket_sub_proof_of_life.connect(r"tcp://127.0.0.1:{}".format(
ct.PROOF_OF_LIFE_FORWARDER_PUBLISH_PORT))
self.socket_sub_proof_of_life.setsockopt(
zmq.SUBSCRIBE, '{}'.format(self.name.replace(' ',
'_')).encode('ascii'))
def remove_from_editor(self):
dpg.remove_alias('verb#{}#{}'.format(self.operation.name,
self.node_index))
if dpg.does_alias_exist('relic#{}#{}'.format(self.operation.name,
self.node_index)):
dpg.remove_alias('relic#{}#{}'.format(self.operation.name,
self.node_index))
dpg.delete_item(self.id)
def get_numbers_of_inputs_and_outputs(self):
for at in self.operation.attribute_types:
if at == 'Input':
self.num_of_inputs = self.num_of_inputs + 1
if at == 'Output':
self.num_of_outputs = self.num_of_outputs + 1
def get_corresponding_operation(self):
name = self.name.split('##')[0]
for op in self.operations_list:
if op.name == name:
self.operation = copy.deepcopy(op)
break
def assign_default_parameters(self):
self.node_parameters = self.operation.parameters_def_values
for default_parameter in self.operation.parameters_def_values:
if type(default_parameter) == list:
self.node_parameters_combos_items.append(default_parameter)
else:
self.node_parameters_combos_items.append(None)
def get_node_index(self):
self.node_index = self.name.split('##')[-1]
def add_topic_in(self, topic):
topic = topic.replace(' ', '_')
for t in self.topics_in:
if t == topic:
return
self.topics_in.append(topic)
def add_topic_out(self, topic):
topic = topic.replace(' ', '_')
for t in self.topics_out:
if topic == t:
return
self.topics_out.append(topic)
def remove_topic_in(self, topic):
if len(self.topics_in) == 1:
if self.topics_in[0] == topic:
self.topics_in[0] = 'NothingIn'
else:
for i, t in enumerate(self.topics_in):
if t == topic:
# self.topics_in[i] = ''
del self.topics_in[i]
break
def remove_topic_out(self, topic):
if len(self.topics_out) == 1:
if self.topics_out[0] == topic:
self.topics_out[0] = 'NothingOut'
else:
for i, t in enumerate(self.topics_out):
if t == topic:
del self.topics_out[i]
break
def update_parameters(self):
if self.socket_pub_parameters is None:
self.initialise_parameters_socket()
attribute_name = 'Parameters' + '##{}##{}'.format(
self.operation.name, self.node_index)
for i, parameter in enumerate(self.operation.parameters):
self.node_parameters[i] = dpg.get_value(
self.parameter_inputs_ids[parameter])
topic = self.operation.name + '##' + self.node_index
topic = topic.replace(' ', '_')
self.socket_pub_parameters.send_string(topic, flags=zmq.SNDMORE)
self.socket_pub_parameters.send_pyobj(self.node_parameters)
def spawn_node_on_editor(self):
self.context = zmq.Context()
self.initialise_parameters_socket()
with dpg.node(label=self.name,
parent=self.parent,
pos=[self.coordinates[0],
self.coordinates[1]]) as self.id:
colour = choose_color_according_to_operations_type(
self.operation.parent_dir)
with dpg.theme() as self.theme_id:
with dpg.theme_component(0):
dpg.add_theme_color(dpg.mvNodeCol_TitleBar,
colour,
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_TitleBarSelected,
colour,
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_TitleBarHovered,
colour,
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_NodeBackgroundSelected,
[120, 120, 120, 255],
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_NodeBackground,
[70, 70, 70, 255],
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_NodeBackgroundHovered,
[80, 80, 80, 255],
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_color(dpg.mvNodeCol_NodeOutline,
[50, 50, 50, 255],
category=dpg.mvThemeCat_Nodes)
dpg.add_theme_style(dpg.mvNodeStyleVar_NodeBorderThickness,
x=4,
category=dpg.mvThemeCat_Nodes)
dpg.bind_item_theme(self.id, self.theme_id)
# Loop through all the attributes defined in the operation (as seen in the *_com.py file) and put them on
# the node
same_line_widget_ids = []
for i, attr in enumerate(self.operation.attributes):
if self.operation.attribute_types[i] == 'Input':
attribute_type = dpg.mvNode_Attr_Input
elif self.operation.attribute_types[i] == 'Output':
attribute_type = dpg.mvNode_Attr_Output
elif self.operation.attribute_types[i] == 'Static':
attribute_type = dpg.mvNode_Attr_Static
attribute_name = attr + '##{}##{}'.format(
self.operation.name, self.node_index)
with dpg.node_attribute(label=attribute_name,
parent=self.id,
attribute_type=attribute_type):
if attribute_type == 1:
dpg.add_spacer()
dpg.add_text(label='##' + attr + ' Name{}##{}'.format(
self.operation.name, self.node_index),
default_value=attr)
if 'Parameters' in attr:
for k, parameter in enumerate(
self.operation.parameters):
if self.operation.parameter_types[k] == 'int':
id = dpg.add_input_int(
label='{}##{}'.format(
parameter, attribute_name),
default_value=self.node_parameters[k],
callback=self.update_parameters,
width=100,
min_clamped=False,
max_clamped=False,
min_value=int(-1e8),
max_value=int(1e8),
on_enter=True)
elif self.operation.parameter_types[k] == 'str':
id = dpg.add_input_text(
label='{}##{}'.format(
parameter, attribute_name),
default_value=self.node_parameters[k],
callback=self.update_parameters,
width=100,
on_enter=True)
elif self.operation.parameter_types[k] == 'float':
id = dpg.add_input_float(
label='{}##{}'.format(
parameter, attribute_name),
default_value=self.node_parameters[k],
callback=self.update_parameters,
width=100,
min_clamped=False,
max_clamped=False,
min_value=-1e10,
max_value=1e10,
on_enter=True)
elif self.operation.parameter_types[k] == 'bool':
id = dpg.add_checkbox(
label='{}##{}'.format(
parameter, attribute_name),
default_value=self.node_parameters[k],
callback=self.update_parameters)
elif self.operation.parameter_types[k] == 'list':
default_value = self.node_parameters[k][0]
if type(self.node_parameters[k]) == str:
default_value = self.node_parameters[k]
id = dpg.add_combo(
label='{}##{}'.format(
parameter, attribute_name),
items=self.node_parameters_combos_items[k],
default_value=default_value,
callback=self.update_parameters,
width=100)
self.parameter_inputs_ids[parameter] = id
dpg.add_spacer(label='##Spacing##' + attribute_name,
indent=3)
# Add the extra input button with its popup window for extra inputs like ssh and verbosity
self.extra_input_window()
def extra_input_window(self):
attr = 'Extra_Input'
attribute_name = attr + '##{}##{}'.format(self.operation.name,
self.node_index)
with dpg.node_attribute(
label=attribute_name,
parent=self.id,
attribute_type=dpg.mvNode_Attr_Static) as attr_id:
image_file = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.path.pardir,
'resources', 'Blue_glass_button_square_34x34.png')
width, height, channels, data = dpg.load_image(image_file)
with dpg.texture_registry():
texture_id = dpg.add_static_texture(width, height, data)
image_button = dpg.add_image_button(
texture_tag=texture_id,
label='##' + attr +
' Name{}##{}'.format(self.operation.name, self.node_index),
callback=self.update_ssh_combo_boxes)
with dpg.window(
label='##Window#Extra input##{}##{}'.format(
self.operation.name, self.node_index),
width=450,
height=250,
pos=[self.coordinates[0] + 400, self.coordinates[1] + 200],
show=False,
popup=True) as self.extra_window_id:
# Add the local ssh input
dpg.add_spacer(height=10)
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_text('SSH local server')
dpg.add_spacer(width=80)
dpg.add_text('SSH remote server')
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
id = dpg.add_combo(
label='##SSH local server##Extra input##{}##{}'.format(
self.operation.name, self.node_index),
items=self.ssh_server_id_and_names,
width=140,
default_value=self.ssh_local_server,
callback=self.assign_local_server)
self.parameter_inputs_ids['SSH local server'] = id
dpg.add_spacer(width=40)
id = dpg.add_combo(
label='##SSH remote server ##Extra input##{}##{}'.
format(self.operation.name, self.node_index),
items=self.ssh_server_id_and_names,
width=140,
default_value=self.ssh_remote_server,
callback=self.assign_remote_server)
self.parameter_inputs_ids['SSH remote server'] = id
dpg.add_spacer(height=10)
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_text(
'Python script of worker process OR Python.exe and script:'
)
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_input_text(
label='##Worker executable##Extra input##{}##{}'.
format(self.operation.name, self.node_index),
width=400,
default_value=self.worker_executable,
callback=self.assign_worker_executable)
# Add the verbocity input
dpg.add_spacer(height=6)
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
attr = 'Log file or Verbosity level:'
attribute_name = attr + '##{}##{}'.format(
self.operation.name, self.node_index)
self.verbosity_id = dpg.add_text(
label='##' + attr + ' Name{}##{}'.format(
self.operation.name, self.node_index),
default_value=attr)
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_input_text(
label='##{}'.format(attribute_name),
default_value=self.com_verbosity,
callback=self.update_verbosity,
width=400,
hint='Log file name or verbosity level integer.',
tag='verb#{}#{}'.format(self.operation.name,
self.node_index))
if importlib.util.find_spec('reliquery') is not None:
# Create the relic input only if reliquery is present
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_text(default_value='Save Relic to directory:')
with dpg.group(horizontal=True):
dpg.add_spacer(width=10)
dpg.add_input_text(
default_value=self.relic_verbosity,
callback=self.update_verbosity,
hint=
'The path where the Relic for this worker process will be saved',
tag='relic#{}#{}'.format(self.operation.name,
self.node_index))
def update_verbosity(self, sender, data):
self.com_verbosity = ''
self.relic_verbosity = ''
if importlib.util.find_spec('reliquery') is not None:
self.relic_verbosity = dpg.get_value('relic#{}#{}'.format(
self.operation.name, self.node_index))
self.com_verbosity = dpg.get_value('verb#{}#{}'.format(
self.operation.name, self.node_index))
self.verbose = '{}||{}'.format(self.com_verbosity,
self.relic_verbosity)
def get_ssh_server_names_and_ids(self):
ssh_info_file = os.path.join(
Path(os.path.dirname(os.path.realpath(__file__))).parent,
'communication', 'ssh_info.json')
with open(ssh_info_file) as f:
ssh_info = json.load(f)
self.ssh_server_id_and_names = ['None']
for id in ssh_info:
self.ssh_server_id_and_names.append(id + ' ' +
ssh_info[id]['Name'])
def update_ssh_combo_boxes(self):
dpg.configure_item(self.extra_window_id, show=True)
self.get_ssh_server_names_and_ids()
if dpg.does_item_exist(self.parameter_inputs_ids['SSH local server']):
dpg.configure_item(self.parameter_inputs_ids['SSH local server'],
items=self.ssh_server_id_and_names)
if dpg.does_item_exist(self.parameter_inputs_ids['SSH remote server']):
dpg.configure_item(self.parameter_inputs_ids['SSH remote server'],
items=self.ssh_server_id_and_names)
def assign_local_server(self, sender, data):
self.ssh_local_server = dpg.get_value(sender)
def assign_remote_server(self, sender, data):
self.ssh_remote_server = dpg.get_value(sender)
def assign_worker_executable(self, sender, data):
self.worker_executable = dpg.get_value(sender)
def start_com_process(self):
self.initialise_proof_of_life_socket()
arguments_list = [
'python', self.operation.executable, self.starting_port
]
num_of_inputs = len(self.topics_in)
num_of_outputs = len(self.topics_out)
arguments_list.append(str(num_of_inputs))
if 'Input' in self.operation.attribute_types:
for topic_in in self.topics_in:
arguments_list.append(topic_in)
arguments_list.append(str(num_of_outputs))
if 'Output' in self.operation.attribute_types:
for topic_out in self.topics_out:
arguments_list.append(topic_out)
arguments_list.append(self.name.replace(" ", "_"))
self.update_verbosity(
None, None
) # This is required to make the debugger work because in debug this callback
# is never called
arguments_list.append(str(self.verbose))
arguments_list.append(self.ssh_local_server.split(
' ')[0]) # pass only the ID part of the 'ID name' string
arguments_list.append(self.ssh_remote_server.split(' ')[0])
arguments_list.append(self.worker_executable)
self.process = subprocess.Popen(
arguments_list, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
print('Started COM {}_{} process with PID = {}'.format(
self.name, self.node_index, self.process.pid))
# Wait until the worker_exec sends a proof_of_life signal (i.e. it is up and running).
self.wait_for_proof_of_life()
# Then update the parameters
self.update_parameters()
self.start_thread_to_send_parameters_multiple_times()
with dpg.theme_component(0, parent=self.theme_id):
dpg.add_theme_color(dpg.mvNodeCol_NodeOutline,
[255, 255, 255, 255],
category=dpg.mvThemeCat_Nodes)
def sending_parameters_multiple_times(self):
for i in range(ct.NUMBER_OF_INITIAL_PARAMETERS_UPDATES):
self.update_parameters()
time.sleep(0.5)
def start_thread_to_send_parameters_multiple_times(self):
thread_parameters = threading.Thread(
target=self.sending_parameters_multiple_times, daemon=True)
thread_parameters.start()
def wait_for_proof_of_life(self):
self.socket_sub_proof_of_life.recv()
self.socket_sub_proof_of_life.recv_string()
def stop_com_process(self):
try:
self.socket_sub_proof_of_life.disconnect(
r"tcp://127.0.0.1:{}".format(
ct.PROOF_OF_LIFE_FORWARDER_PUBLISH_PORT))
self.socket_sub_proof_of_life.close()
except:
pass
if platform.system() == 'Windows':
self.process.send_signal(signal.CTRL_BREAK_EVENT)
elif platform.system() == 'Linux':
self.process.terminate()
time.sleep(0.5)
self.process.kill()
self.process = None
with dpg.theme_component(0, parent=self.theme_id):
dpg.add_theme_color(dpg.mvNodeCol_NodeOutline, [50, 50, 50, 255],
category=dpg.mvThemeCat_Nodes)
