class SourceCom: def __init__(self, sending_topics, parameters_topic, port, worker_exec, verbose='||', ssh_local_server_id='None', ssh_remote_server_id='None', outputs=None): self.sending_topics = sending_topics self.parameters_topic = parameters_topic self.pull_data_port = port self.heartbeat_port = str(int(self.pull_data_port) + 1) self.worker_exec = worker_exec self.index = 0 self.time = int(1000000 * time.perf_counter()) self.previous_time = self.time self.verbose, self.relic = self.define_verbosity_and_relic(verbose) self.all_loops_running = True self.ssh_com = SSHCom(self.worker_exec, ssh_local_server_id, ssh_remote_server_id) self.outputs = outputs self.port_pub = ct.DATA_FORWARDER_SUBMIT_PORT self.context = None self.socket_pub_data = None self.socket_pull_data = None self.stream_pull_data = None self.socket_push_heartbeat = None self.average_sending_time = 0 # If self.verbose is a string it is the file name to log things in. If it is an int it is the level of the verbosity self.logger = None if self.verbose != 0: try: self.verbose = int(self.verbose) except: log_file_name = gu.add_timestamp_to_filename(self.verbose, datetime.now()) self.logger = gu.setup_logger('Source', log_file_name) self.logger.info('Index of data packet : Computer Time Data Out') self.verbose = False def connect_sockets(self): """ Start the required sockets to communicate with the link forwarder and the source_com processes :return: Nothing """ if self.verbose: print('Starting Source Node with PID = {}'.format(os.getpid())) self.context = zmq.Context() # Socket for pulling the data from the worker_exec self.socket_pull_data = Socket(self.context, zmq.PULL) self.socket_pull_data.setsockopt(zmq.LINGER, 0) self.socket_pull_data.set_hwm(1) self.socket_pull_data.connect(r"tcp://127.0.0.1:{}".format(self.pull_data_port)) self.stream_pull_data = zmqstream.ZMQStream(self.socket_pull_data) self.stream_pull_data.on_recv(self.on_receive_data_from_worker) # Socket for publishing the data to the data forwarder self.socket_pub_data = Socket(self.context, zmq.PUB) self.socket_pub_data.setsockopt(zmq.LINGER, 0) self.socket_pub_data.set_hwm(len(self.sending_topics)) self.socket_pub_data.connect(r"tcp://127.0.0.1:{}".format(self.port_pub)) # Socket for publishing the heartbeat to the worker_exec self.socket_push_heartbeat = self.context.socket(zmq.PUSH) self.socket_push_heartbeat.setsockopt(zmq.LINGER, 0) self.socket_push_heartbeat.bind(r'tcp://*:{}'.format(self.heartbeat_port)) self.socket_push_heartbeat.set_hwm(1) def define_verbosity_and_relic(self, verbosity_string): """ Splits the string that comes from the Node as verbosity_string into the string (or int) for the logging/printing (self.verbose) and the string that carries the path where the relic is to be saved. The self.relic is then passed to the worker process :param verbosity_string: The string with syntax verbosity||relic :return: (int)str vebrose, str relic """ if verbosity_string != '': verbosity, relic = verbosity_string.split('||') if relic == '': relic = '_' if verbosity == '': return 0, relic else: return verbosity, relic else: return 0, '' def on_receive_data_from_worker(self, msg): """ The callback that runs every time link is received from the worker_exec process. It takes the link and passes it onto the link forwarder :param msg: The link packet (carrying the actual link (np array)) :return: """ # A specific worker with multiple outputs should send from its infinite loop a message with multiple parts # (using multiple send_array(data, flags=zmq.SNDMORE) commands). For an example see how the transform_worker # sends data to the com from its data_callback function # TODO The bellow will not work for multiple outputs. I have to find out how many times a callback is called # when data are send with SNDMORE flag !!! ignoring_outputs = [False] * len(self.outputs) new_message_data = [] if len(self.outputs) > 1: for i in range(len(self.outputs)): array_data = Socket.reconstruct_array_from_bytes_message(msg[i]) new_message_data.append(array_data) if type(array_data[0]) == np.str_: if array_data[0] == ct.IGNORE: ignoring_outputs[i] = True else: array_data = Socket.reconstruct_array_from_bytes_message(msg) new_message_data.append(array_data) if type(array_data[0]) == np.str_: if array_data[0] == ct.IGNORE: ignoring_outputs[0] = True self.time = int(1000000 * time.perf_counter()) self.index = self.index + 1 # Publish the results. Each array in the list of arrays is published to its own sending topic # (matched by order) for i, st in enumerate(self.sending_topics): for k, output in enumerate(self.outputs): if output.replace(' ', '_') in st.split('##')[0]: break if ignoring_outputs[k] is False: self.socket_pub_data.send("{}".format(st).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send("{}".format(self.index).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send("{}".format(self.time).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send_array(new_message_data[k], copy=False) # This delay is critical to get single output to multiple inputs to work! gu.accurate_delay(ct.DELAY_BETWEEN_SENDING_DATA_TO_NEXT_NODE_MILLISECONDS) if self.verbose: dt = self.time - self.previous_time if self.index > 3: self.average_sending_time = self.average_sending_time * (self.index - 1) / self.index + dt / self.index print('----------') print("Source with topic {} sending packet with data_index {} at time {}".format(self.sending_topics[i], self.index, self.time)) print('Time Diff between packages = {}. Average package sending time = {} ms'.format(dt/1000, self.average_sending_time / 1000)) if self.logger: self.logger.info('{} : {}'.format(self.index, datetime.now())) self.previous_time = self.time def heartbeat_loop(self): """ Sending every ct.HEARTBEAT_RATE a 'PULSE' to the worker_exec so that it stays alive :return: Nothing """ while self.all_loops_running: self.socket_push_heartbeat.send_string('PULSE') time.sleep(ct.HEARTBEAT_RATE) def start_heartbeat_thread(self): """ Starts the daemon thread that runs the self.heartbeat loop :return: Nothing """ heartbeat_thread = threading.Thread(target=self.heartbeat_loop, daemon=True) heartbeat_thread.start() def start_worker_process(self): """ Starts the worker_exec process and then sends the parameters as are currently on the node to the process The pull_data_port of the worker_exec needs to be the push_data_port of the com (obviously). The way the arguments are structured is defined by the way they are read by the process. For that see general_utilities.parse_arguments_to_worker :return: Nothing """ if 'python' in self.worker_exec or '.py' not in self.worker_exec: arguments_list = [self.worker_exec] else: arguments_list = ['python'] arguments_list.append(self.worker_exec) arguments_list.append(str(self.pull_data_port)) arguments_list.append(str(self.parameters_topic)) arguments_list.append(str(0)) arguments_list.append(str(len(self.sending_topics))) arguments_list.append(self.relic) arguments_list = self.ssh_com.add_local_server_info_to_arguments(arguments_list) worker_pid = self.ssh_com.start_process(arguments_list) self.ssh_com.connect_socket_to_remote(self.socket_pull_data, r"tcp://127.0.0.1:{}".format(self.pull_data_port)) def start_ioloop(self): """ Starts the ioloop of the zmqstream :return: Nothing """ ioloop.IOLoop.instance().start() def on_kill(self, signal, frame): """ The function that is called when the parent process sends a SIGBREAK (windows) or SIGTERM (linux) signal. It needs signal and frame as parameters :param signal: The signal received :param frame: I haven't got a clue :return: Nothing """ try: self.all_loops_running = False self.stream_pull_data.close(linger=0) self.socket_pull_data.close() self.socket_pub_data.close() self.socket_push_heartbeat.close() except Exception as e: print('Trying to kill Source com {} failed with error: {}'.format(self.sending_topics[0], e)) finally: self.context.term()
class SinkCom: def __init__(self, receiving_topics, parameters_topic, push_port, worker_exec, verbose=True, ssh_local_server_id='None', ssh_remote_server_id='None'): self.receiving_topics = receiving_topics self.parameters_topic = parameters_topic self.push_data_port = push_port self.pull_data_port = str(int(self.push_data_port) + 1) self.push_heartbeat_port = str(int(self.push_data_port) + 2) self.worker_exec = worker_exec self.verbose = verbose self.verbose, self.relic = self.define_verbosity_and_relic(verbose) self.all_loops_running = True self.ssh_com = SSHCom(self.worker_exec, ssh_local_server_id, ssh_remote_server_id) self.port_sub_data = ct.DATA_FORWARDER_PUBLISH_PORT self.port_pub_parameters = ct.PARAMETERS_FORWARDER_SUBMIT_PORT self.poller = zmq.Poller() self.context = None self.socket_sub_data = None self.stream_sub = None self.socket_push_data = None self.socket_pull_data = None self.socket_push_heartbeat = None self.index = 0 # If self.verbose is a string it is the file name to log things in. If it is an int it is the level of the verbosity self.logger = None if self.verbose != 0: try: self.verbose = int(self.verbose) except: log_file_name = gu.add_timestamp_to_filename( self.verbose, datetime.now()) self.logger = gu.setup_logger('Sink', log_file_name) self.logger.info( 'Index of data packet given : Index of data packet received: Topic : Computer Time' ) self.verbose = False atexit.register(self.on_kill, None, None) signal.signal(signal.SIGTERM, self.on_kill) def connect_sockets(self): """ Start the required sockets to communicate with the link forwarder and the worker_com processes :return: Nothing """ if self.verbose: print('Starting Sink Node with PID = {}'.format(os.getpid())) self.context = zmq.Context() # Socket for subscribing to data from nodes connected to the input self.socket_sub_data = Socket(self.context, zmq.SUB) self.socket_sub_data.setsockopt(zmq.LINGER, 0) self.socket_sub_data.set_hwm(len(self.receiving_topics)) self.socket_sub_data.connect("tcp://127.0.0.1:{}".format( self.port_sub_data)) for rt in self.receiving_topics: self.socket_sub_data.setsockopt(zmq.SUBSCRIBE, rt.encode('ascii')) self.poller.register(self.socket_sub_data, zmq.POLLIN) # Socket for pushing the data to the worker_exec self.socket_push_data = Socket(self.context, zmq.PUSH) self.socket_push_data.setsockopt(zmq.LINGER, 0) self.socket_push_data.set_hwm(1) self.socket_push_data.bind(r"tcp://*:{}".format(self.push_data_port)) # Socket for pulling the end of worker function signal from the worker_exec self.socket_pull_data = Socket(self.context, zmq.PULL) self.socket_pull_data.setsockopt(zmq.LINGER, 0) self.socket_pull_data.set_hwm(1) self.socket_pull_data.connect(r"tcp://127.0.0.1:{}".format( self.pull_data_port)) self.poller.register(self.socket_pull_data, zmq.POLLIN) # Socket for pushing the heartbeat to the worker_exec self.socket_push_heartbeat = self.context.socket(zmq.PUSH) self.socket_push_heartbeat.setsockopt(zmq.LINGER, 0) self.socket_push_heartbeat.bind(r'tcp://*:{}'.format( self.push_heartbeat_port)) self.socket_push_heartbeat.set_hwm(1) def define_verbosity_and_relic(self, verbosity_string): """ Splits the string that comes from the Node as verbosity_string into the string (or int) for the logging/printing (self.verbose) and the string that carries the path where the relic is to be saved. The self.relic is then passed to the worker process :param verbosity_string: The string with syntax verbosity||relic :return: (int)str vebrose, str relic """ if verbosity_string != '': verbosity, relic = verbosity_string.split('||') if relic == '': relic = '_' if verbosity == '': return 0, relic else: return verbosity, relic else: return 0, '' def heartbeat_loop(self): """ The loop that send a 'PULSE' heartbeat to the worker_exec process to keep it alive (every ct.HEARTBEAT_RATE seconds) :return: Nothing """ while self.all_loops_running: self.socket_push_heartbeat.send_string('PULSE') time.sleep(ct.HEARTBEAT_RATE) def start_heartbeat_thread(self): """ The daemon thread that runs the infinite heartbeat_loop :return: Noting """ heartbeat_thread = threading.Thread(target=self.heartbeat_loop, daemon=True) heartbeat_thread.start() def start_worker(self): """ Starts the worker_exec process and then sends the parameters as are currently on the node to the process The pull_data_port of the worker_exec needs to be the push_data_port of the com (obviously). The way the arguments are structured is defined by the way they are read by the process. For that see general_utilities.parse_arguments_to_worker :return: Nothing """ if 'python' in self.worker_exec or '.py' not in self.worker_exec: arguments_list = [self.worker_exec] else: arguments_list = ['python'] arguments_list.append(self.worker_exec) arguments_list.append(str(self.push_data_port)) arguments_list.append(str(self.parameters_topic)) arguments_list.append(str(len(self.receiving_topics))) for i in range(len(self.receiving_topics)): arguments_list.append(self.receiving_topics[i]) arguments_list.append(str(0)) arguments_list.append(str(self.relic)) arguments_list = self.ssh_com.add_local_server_info_to_arguments( arguments_list) worker_pid = self.ssh_com.start_process(arguments_list) self.ssh_com.connect_socket_to_remote( self.socket_pull_data, r"tcp://127.0.0.1:{}".format(self.pull_data_port)) def get_sub_data(self): """ Gets the link from the forwarder. It assumes that each message has four parts: The topic The data_index, an int that increases by one for every message the previous node sends The data_time, the time.perf_counter() result at the time the previous node send its message The messagedata, the array of link :return: Nothing """ prev_topic = self.socket_sub_data.recv() prev_data_index = self.socket_sub_data.recv() prev_data_time = self.socket_sub_data.recv() prev_messagedata = self.socket_sub_data.recv_array() # The following while ensures that the sink works only on the the latest available # message from the previous node. If the sink is too slow compared to the input node # this while throws all past messages away. while prev_topic: topic = prev_topic data_index = prev_data_index data_time = prev_data_time messagedata = prev_messagedata try: prev_topic = self.socket_sub_data.recv(zmq.NOBLOCK) prev_data_index = self.socket_sub_data.recv(zmq.NOBLOCK) prev_data_time = self.socket_sub_data.recv(zmq.NOBLOCK) prev_messagedata = self.socket_sub_data.recv_array(zmq.NOBLOCK) except: prev_topic = None pass return topic, data_index, data_time, messagedata def start_ioloop(self): """ Start the io loop for the sink node. It reads the link from the previous node's _com process, pushes it to the worker_com process, waits for the results, grabs the resulting link from the worker_com process and publishes the transformed link to the link forwarder with this nodes' topic :return: Nothing """ while self.all_loops_running: t1 = time.perf_counter() try: # The timeout=1 means things coming in faster than 1000Hz will be lost but if timeout is set to 0 then # the CPU utilization goes to around 10% which quickly kills the CPU (if there are 2 or 3 Sinks in the # pipeline) sockets_in = dict(self.poller.poll(timeout=1)) while not sockets_in: sockets_in = dict(self.poller.poll(timeout=1)) if self.socket_sub_data in sockets_in and sockets_in[ self.socket_sub_data] == zmq.POLLIN: topic, data_index, data_time, messagedata = self.get_sub_data( ) sockets_in = dict(self.poller.poll(timeout=1)) if self.verbose: print( "oooo Sink from {}, data_index {} at time {} s oooo" .format(topic, data_index, data_time)) # Send link to be transformed to the worker_exec self.socket_push_data.send(topic, flags=zmq.SNDMORE) self.socket_push_data.send_array(messagedata, copy=False) t2 = time.perf_counter() # Get the end of worker function link (wait for the socket_pull_data to get some link from the worker_exec) sockets_in = dict(self.poller.poll(timeout=None)) self.socket_pull_data.recv() t3 = time.perf_counter() if self.verbose: print( "---Time to Transport link from previous com to worker_exec = {} ms" .format((t2 - t1) * 1000)) print("---Time to to finish with the worker_exec = {} ms". format((t3 - t2) * 1000)) print('=============================') if self.logger: self.logger.info('{} : {} : {} : {}'.format( self.index, data_index, topic, datetime.now())) self.index += 1 except: pass def on_kill(self, signal, frame): try: self.all_loops_running = False self.poller.unregister(socket=self.socket_sub_data) self.poller.unregister(socket=self.socket_pull_data) self.socket_sub_data.close() self.socket_push_data.close() self.socket_pull_data.close() self.socket_push_heartbeat.close() except Exception as e: print('Trying to kill Sink com {} failed with error: {}'.format( self.worker_exec, e)) finally: self.context.term()
class Node: def __init__(self, name, parent): self.id = None self.name = name self.parent = parent self.operation = None self.node_index = None self.process = None self.topics_out = [] self.topics_in = [] self.links_list = [] self.starting_port = None self.num_of_inputs = 0 self.num_of_outputs = 0 self.coordinates = [100, 100] self.node_parameters = None self.node_parameters_combos_items = [] self.parameter_inputs_ids = {} self.com_verbosity = '' self.relic_verbosity = '' self.verbose = '' self.context = None self.socket_pub_parameters = None self.socket_sub_proof_of_life = None self.theme_id = None self.extra_window_id = None self.operations_list = op.operations_list self.get_corresponding_operation() self.get_node_index() self.assign_default_parameters() self.get_numbers_of_inputs_and_outputs() #self.generate_default_topics() self.ssh_server_id_and_names = None self.get_ssh_server_names_and_ids() self.ssh_local_server = self.ssh_server_id_and_names[0] self.ssh_remote_server = self.ssh_server_id_and_names[0] self.worker_executable = self.operation.worker_exec def get_attribute_order(self, type): number_of_static_attrs = 0 for at in self.operation.attribute_types: if 'Static' in at: number_of_static_attrs += 1 number_of_input_attrs = 0 number_of_output_attrs = 0 order = {} for at, attr_order_num in zip(self.operation.attribute_types, range(len(self.operation.attributes))): if type == 'Input' and type in at: order[self.operation. attributes[attr_order_num]] = number_of_input_attrs number_of_input_attrs += 1 if type == 'Output' and type in at: order[self.operation. attributes[attr_order_num]] = number_of_output_attrs number_of_output_attrs += 1 return order def initialise_parameters_socket(self): if self.context is None: self.context = zmq.Context() self.socket_pub_parameters = Socket(self.context, zmq.PUB) self.socket_pub_parameters.setsockopt(zmq.LINGER, 0) self.socket_pub_parameters.connect(r"tcp://127.0.0.1:{}".format( ct.PARAMETERS_FORWARDER_SUBMIT_PORT)) def initialise_proof_of_life_socket(self): if self.context is None: self.context = zmq.Context() self.socket_sub_proof_of_life = self.context.socket(zmq.SUB) self.socket_sub_proof_of_life.setsockopt(zmq.LINGER, 0) self.socket_sub_proof_of_life.connect(r"tcp://127.0.0.1:{}".format( ct.PROOF_OF_LIFE_FORWARDER_PUBLISH_PORT)) self.socket_sub_proof_of_life.setsockopt( zmq.SUBSCRIBE, '{}'.format(self.name.replace(' ', '_')).encode('ascii')) def remove_from_editor(self): dpg.remove_alias('verb#{}#{}'.format(self.operation.name, self.node_index)) if dpg.does_alias_exist('relic#{}#{}'.format(self.operation.name, self.node_index)): dpg.remove_alias('relic#{}#{}'.format(self.operation.name, self.node_index)) dpg.delete_item(self.id) def get_numbers_of_inputs_and_outputs(self): for at in self.operation.attribute_types: if at == 'Input': self.num_of_inputs = self.num_of_inputs + 1 if at == 'Output': self.num_of_outputs = self.num_of_outputs + 1 def get_corresponding_operation(self): name = self.name.split('##')[0] for op in self.operations_list: if op.name == name: self.operation = copy.deepcopy(op) break def assign_default_parameters(self): self.node_parameters = self.operation.parameters_def_values for default_parameter in self.operation.parameters_def_values: if type(default_parameter) == list: self.node_parameters_combos_items.append(default_parameter) else: self.node_parameters_combos_items.append(None) def get_node_index(self): self.node_index = self.name.split('##')[-1] def add_topic_in(self, topic): topic = topic.replace(' ', '_') for t in self.topics_in: if t == topic: return self.topics_in.append(topic) def add_topic_out(self, topic): topic = topic.replace(' ', '_') for t in self.topics_out: if topic == t: return self.topics_out.append(topic) def remove_topic_in(self, topic): if len(self.topics_in) == 1: if self.topics_in[0] == topic: self.topics_in[0] = 'NothingIn' else: for i, t in enumerate(self.topics_in): if t == topic: # self.topics_in[i] = '' del self.topics_in[i] break def remove_topic_out(self, topic): if len(self.topics_out) == 1: if self.topics_out[0] == topic: self.topics_out[0] = 'NothingOut' else: for i, t in enumerate(self.topics_out): if t == topic: del self.topics_out[i] break def update_parameters(self): if self.socket_pub_parameters is None: self.initialise_parameters_socket() attribute_name = 'Parameters' + '##{}##{}'.format( self.operation.name, self.node_index) for i, parameter in enumerate(self.operation.parameters): self.node_parameters[i] = dpg.get_value( self.parameter_inputs_ids[parameter]) topic = self.operation.name + '##' + self.node_index topic = topic.replace(' ', '_') self.socket_pub_parameters.send_string(topic, flags=zmq.SNDMORE) self.socket_pub_parameters.send_pyobj(self.node_parameters) def spawn_node_on_editor(self): self.context = zmq.Context() self.initialise_parameters_socket() with dpg.node(label=self.name, parent=self.parent, pos=[self.coordinates[0], self.coordinates[1]]) as self.id: colour = choose_color_according_to_operations_type( self.operation.parent_dir) with dpg.theme() as self.theme_id: with dpg.theme_component(0): dpg.add_theme_color(dpg.mvNodeCol_TitleBar, colour, category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_TitleBarSelected, colour, category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_TitleBarHovered, colour, category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_NodeBackgroundSelected, [120, 120, 120, 255], category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_NodeBackground, [70, 70, 70, 255], category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_NodeBackgroundHovered, [80, 80, 80, 255], category=dpg.mvThemeCat_Nodes) dpg.add_theme_color(dpg.mvNodeCol_NodeOutline, [50, 50, 50, 255], category=dpg.mvThemeCat_Nodes) dpg.add_theme_style(dpg.mvNodeStyleVar_NodeBorderThickness, x=4, category=dpg.mvThemeCat_Nodes) dpg.bind_item_theme(self.id, self.theme_id) # Loop through all the attributes defined in the operation (as seen in the *_com.py file) and put them on # the node same_line_widget_ids = [] for i, attr in enumerate(self.operation.attributes): if self.operation.attribute_types[i] == 'Input': attribute_type = dpg.mvNode_Attr_Input elif self.operation.attribute_types[i] == 'Output': attribute_type = dpg.mvNode_Attr_Output elif self.operation.attribute_types[i] == 'Static': attribute_type = dpg.mvNode_Attr_Static attribute_name = attr + '##{}##{}'.format( self.operation.name, self.node_index) with dpg.node_attribute(label=attribute_name, parent=self.id, attribute_type=attribute_type): if attribute_type == 1: dpg.add_spacer() dpg.add_text(label='##' + attr + ' Name{}##{}'.format( self.operation.name, self.node_index), default_value=attr) if 'Parameters' in attr: for k, parameter in enumerate( self.operation.parameters): if self.operation.parameter_types[k] == 'int': id = dpg.add_input_int( label='{}##{}'.format( parameter, attribute_name), default_value=self.node_parameters[k], callback=self.update_parameters, width=100, min_clamped=False, max_clamped=False, min_value=int(-1e8), max_value=int(1e8), on_enter=True) elif self.operation.parameter_types[k] == 'str': id = dpg.add_input_text( label='{}##{}'.format( parameter, attribute_name), default_value=self.node_parameters[k], callback=self.update_parameters, width=100, on_enter=True) elif self.operation.parameter_types[k] == 'float': id = dpg.add_input_float( label='{}##{}'.format( parameter, attribute_name), default_value=self.node_parameters[k], callback=self.update_parameters, width=100, min_clamped=False, max_clamped=False, min_value=-1e10, max_value=1e10, on_enter=True) elif self.operation.parameter_types[k] == 'bool': id = dpg.add_checkbox( label='{}##{}'.format( parameter, attribute_name), default_value=self.node_parameters[k], callback=self.update_parameters) elif self.operation.parameter_types[k] == 'list': default_value = self.node_parameters[k][0] if type(self.node_parameters[k]) == str: default_value = self.node_parameters[k] id = dpg.add_combo( label='{}##{}'.format( parameter, attribute_name), items=self.node_parameters_combos_items[k], default_value=default_value, callback=self.update_parameters, width=100) self.parameter_inputs_ids[parameter] = id dpg.add_spacer(label='##Spacing##' + attribute_name, indent=3) # Add the extra input button with its popup window for extra inputs like ssh and verbosity self.extra_input_window() def extra_input_window(self): attr = 'Extra_Input' attribute_name = attr + '##{}##{}'.format(self.operation.name, self.node_index) with dpg.node_attribute( label=attribute_name, parent=self.id, attribute_type=dpg.mvNode_Attr_Static) as attr_id: image_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), os.path.pardir, 'resources', 'Blue_glass_button_square_34x34.png') width, height, channels, data = dpg.load_image(image_file) with dpg.texture_registry(): texture_id = dpg.add_static_texture(width, height, data) image_button = dpg.add_image_button( texture_tag=texture_id, label='##' + attr + ' Name{}##{}'.format(self.operation.name, self.node_index), callback=self.update_ssh_combo_boxes) with dpg.window( label='##Window#Extra input##{}##{}'.format( self.operation.name, self.node_index), width=450, height=250, pos=[self.coordinates[0] + 400, self.coordinates[1] + 200], show=False, popup=True) as self.extra_window_id: # Add the local ssh input dpg.add_spacer(height=10) with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_text('SSH local server') dpg.add_spacer(width=80) dpg.add_text('SSH remote server') with dpg.group(horizontal=True): dpg.add_spacer(width=10) id = dpg.add_combo( label='##SSH local server##Extra input##{}##{}'.format( self.operation.name, self.node_index), items=self.ssh_server_id_and_names, width=140, default_value=self.ssh_local_server, callback=self.assign_local_server) self.parameter_inputs_ids['SSH local server'] = id dpg.add_spacer(width=40) id = dpg.add_combo( label='##SSH remote server ##Extra input##{}##{}'. format(self.operation.name, self.node_index), items=self.ssh_server_id_and_names, width=140, default_value=self.ssh_remote_server, callback=self.assign_remote_server) self.parameter_inputs_ids['SSH remote server'] = id dpg.add_spacer(height=10) with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_text( 'Python script of worker process OR Python.exe and script:' ) with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_input_text( label='##Worker executable##Extra input##{}##{}'. format(self.operation.name, self.node_index), width=400, default_value=self.worker_executable, callback=self.assign_worker_executable) # Add the verbocity input dpg.add_spacer(height=6) with dpg.group(horizontal=True): dpg.add_spacer(width=10) attr = 'Log file or Verbosity level:' attribute_name = attr + '##{}##{}'.format( self.operation.name, self.node_index) self.verbosity_id = dpg.add_text( label='##' + attr + ' Name{}##{}'.format( self.operation.name, self.node_index), default_value=attr) with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_input_text( label='##{}'.format(attribute_name), default_value=self.com_verbosity, callback=self.update_verbosity, width=400, hint='Log file name or verbosity level integer.', tag='verb#{}#{}'.format(self.operation.name, self.node_index)) if importlib.util.find_spec('reliquery') is not None: # Create the relic input only if reliquery is present with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_text(default_value='Save Relic to directory:') with dpg.group(horizontal=True): dpg.add_spacer(width=10) dpg.add_input_text( default_value=self.relic_verbosity, callback=self.update_verbosity, hint= 'The path where the Relic for this worker process will be saved', tag='relic#{}#{}'.format(self.operation.name, self.node_index)) def update_verbosity(self, sender, data): self.com_verbosity = '' self.relic_verbosity = '' if importlib.util.find_spec('reliquery') is not None: self.relic_verbosity = dpg.get_value('relic#{}#{}'.format( self.operation.name, self.node_index)) self.com_verbosity = dpg.get_value('verb#{}#{}'.format( self.operation.name, self.node_index)) self.verbose = '{}||{}'.format(self.com_verbosity, self.relic_verbosity) def get_ssh_server_names_and_ids(self): ssh_info_file = os.path.join( Path(os.path.dirname(os.path.realpath(__file__))).parent, 'communication', 'ssh_info.json') with open(ssh_info_file) as f: ssh_info = json.load(f) self.ssh_server_id_and_names = ['None'] for id in ssh_info: self.ssh_server_id_and_names.append(id + ' ' + ssh_info[id]['Name']) def update_ssh_combo_boxes(self): dpg.configure_item(self.extra_window_id, show=True) self.get_ssh_server_names_and_ids() if dpg.does_item_exist(self.parameter_inputs_ids['SSH local server']): dpg.configure_item(self.parameter_inputs_ids['SSH local server'], items=self.ssh_server_id_and_names) if dpg.does_item_exist(self.parameter_inputs_ids['SSH remote server']): dpg.configure_item(self.parameter_inputs_ids['SSH remote server'], items=self.ssh_server_id_and_names) def assign_local_server(self, sender, data): self.ssh_local_server = dpg.get_value(sender) def assign_remote_server(self, sender, data): self.ssh_remote_server = dpg.get_value(sender) def assign_worker_executable(self, sender, data): self.worker_executable = dpg.get_value(sender) def start_com_process(self): self.initialise_proof_of_life_socket() arguments_list = [ 'python', self.operation.executable, self.starting_port ] num_of_inputs = len(self.topics_in) num_of_outputs = len(self.topics_out) arguments_list.append(str(num_of_inputs)) if 'Input' in self.operation.attribute_types: for topic_in in self.topics_in: arguments_list.append(topic_in) arguments_list.append(str(num_of_outputs)) if 'Output' in self.operation.attribute_types: for topic_out in self.topics_out: arguments_list.append(topic_out) arguments_list.append(self.name.replace(" ", "_")) self.update_verbosity( None, None ) # This is required to make the debugger work because in debug this callback # is never called arguments_list.append(str(self.verbose)) arguments_list.append(self.ssh_local_server.split( ' ')[0]) # pass only the ID part of the 'ID name' string arguments_list.append(self.ssh_remote_server.split(' ')[0]) arguments_list.append(self.worker_executable) self.process = subprocess.Popen( arguments_list, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) print('Started COM {}_{} process with PID = {}'.format( self.name, self.node_index, self.process.pid)) # Wait until the worker_exec sends a proof_of_life signal (i.e. it is up and running). self.wait_for_proof_of_life() # Then update the parameters self.update_parameters() self.start_thread_to_send_parameters_multiple_times() with dpg.theme_component(0, parent=self.theme_id): dpg.add_theme_color(dpg.mvNodeCol_NodeOutline, [255, 255, 255, 255], category=dpg.mvThemeCat_Nodes) def sending_parameters_multiple_times(self): for i in range(ct.NUMBER_OF_INITIAL_PARAMETERS_UPDATES): self.update_parameters() time.sleep(0.5) def start_thread_to_send_parameters_multiple_times(self): thread_parameters = threading.Thread( target=self.sending_parameters_multiple_times, daemon=True) thread_parameters.start() def wait_for_proof_of_life(self): self.socket_sub_proof_of_life.recv() self.socket_sub_proof_of_life.recv_string() def stop_com_process(self): try: self.socket_sub_proof_of_life.disconnect( r"tcp://127.0.0.1:{}".format( ct.PROOF_OF_LIFE_FORWARDER_PUBLISH_PORT)) self.socket_sub_proof_of_life.close() except: pass if platform.system() == 'Windows': self.process.send_signal(signal.CTRL_BREAK_EVENT) elif platform.system() == 'Linux': self.process.terminate() time.sleep(0.5) self.process.kill() self.process = None with dpg.theme_component(0, parent=self.theme_id): dpg.add_theme_color(dpg.mvNodeCol_NodeOutline, [50, 50, 50, 255], category=dpg.mvThemeCat_Nodes)