def on_receive_data_from_worker(self, msg): """ The callback that runs every time link is received from the worker_exec process. It takes the link and passes it onto the link forwarder :param msg: The link packet (carrying the actual link (np array)) :return: """ # A specific worker with multiple outputs should send from its infinite loop a message with multiple parts # (using multiple send_array(data, flags=zmq.SNDMORE) commands). For an example see how the transform_worker # sends data to the com from its data_callback function # TODO The bellow will not work for multiple outputs. I have to find out how many times a callback is called # when data are send with SNDMORE flag !!! ignoring_outputs = [False] * len(self.outputs) new_message_data = [] if len(self.outputs) > 1: for i in range(len(self.outputs)): array_data = Socket.reconstruct_array_from_bytes_message(msg[i]) new_message_data.append(array_data) if type(array_data[0]) == np.str_: if array_data[0] == ct.IGNORE: ignoring_outputs[i] = True else: array_data = Socket.reconstruct_array_from_bytes_message(msg) new_message_data.append(array_data) if type(array_data[0]) == np.str_: if array_data[0] == ct.IGNORE: ignoring_outputs[0] = True self.time = int(1000000 * time.perf_counter()) self.index = self.index + 1 # Publish the results. Each array in the list of arrays is published to its own sending topic # (matched by order) for i, st in enumerate(self.sending_topics): for k, output in enumerate(self.outputs): if output.replace(' ', '_') in st.split('##')[0]: break if ignoring_outputs[k] is False: self.socket_pub_data.send("{}".format(st).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send("{}".format(self.index).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send("{}".format(self.time).encode('ascii'), flags=zmq.SNDMORE) self.socket_pub_data.send_array(new_message_data[k], copy=False) # This delay is critical to get single output to multiple inputs to work! gu.accurate_delay(ct.DELAY_BETWEEN_SENDING_DATA_TO_NEXT_NODE_MILLISECONDS) if self.verbose: dt = self.time - self.previous_time if self.index > 3: self.average_sending_time = self.average_sending_time * (self.index - 1) / self.index + dt / self.index print('----------') print("Source with topic {} sending packet with data_index {} at time {}".format(self.sending_topics[i], self.index, self.time)) print('Time Diff between packages = {}. Average package sending time = {} ms'.format(dt/1000, self.average_sending_time / 1000)) if self.logger: self.logger.info('{} : {}'.format(self.index, datetime.now())) self.previous_time = self.time
def visualise(msg, parameters): global visualisation_type global visualisation_on global data message = msg[1:] # data[0] is the topic data = Socket.reconstruct_array_from_bytes_message(message) if parameters is not None: visualisation_on = parameters[0] try: if visualisation_type == 'Image': show_image() elif visualisation_type == 'Value': update_dpg_gui() elif visualisation_type == 'Single Pane Plot': update_dpg_gui() elif visualisation_type == 'Multi Pane Plot': update_dpg_gui() elif visualisation_type == 'Histogram': pass except Exception as e: print(e) return [data]
def work_function(data, parameters): global global_var_1 global global_var_2 global global_var_3 global global_var_4 global vis ''' # This is a 2nd way to initialise the parameters if no initialisation function is used. global need_parameters if need_parameters: try: global_var_1 = parameters[1] global_var_2 = parameters[2] global_var_3 = parameters[3] global_var_1 = parameters[4] need_parameters = True except: pass else: # Do the rest of the code in here so that it doesn't run if the parameters are not set ''' # If any parameters need to be updated during runtime then do that here, e.g. # Also update the visualisation parameter. This allows to turn on and off the visualisation window during # run time try: global_var_1 = parameters[1] vis.visualisation_on = parameters[0] except: pass # In the case of multiple inputs the topic will tell you which input the message has come from. The topic is a # string that is formatted as follows: # previous_node_name##previous_node_index##previous_node_output_name -> this_none_name##this_node_index##this_node_input_name # so you can see which input the data is coming from by looking at the ##this_node_input_name part. Also although # the names of the inputs and outputs can have spaces, these become underscores in the names of the topics. topic = data[0] print( topic ) # prints will not work if the operation is running on a different computer. # The message is a numpy array send in two parts, a header dic (as bytes0 with the array's info and list of bytes # that carry the array's payload. message = data[1:] message = Socket.reconstruct_array_from_bytes_message(message)[ 0] # This is needed to reconstruct the message # that comes in into the numpy array that it is. # Now do stuff print(message.shape) # Whatever data the Node must visualise should be put in the vis.visualised_data variable vis.visualised_data = np.random.random((100, 100))
def work_function(data, parameters): global vis global df try: vis.visualisation_on = parameters[0] except: pass topic = data[0] message = data[1:] message = Socket.reconstruct_array_from_bytes_message(message) #print('--- Message in Save DF: {}'.format(message)) time = datetime.now() row = pd.DataFrame([message], columns=df.columns, index=[time]) #print('--- DF Row: {}'.format(row)) df = pd.concat([df, row], ignore_index=False)
def work_function(data, parameters): global function topic = data[0].decode('utf-8') message = data[1:] # data[0] is the topic input_data = Socket.reconstruct_array_from_bytes_message(message) result = function(topic, input_data) if type(result) == np.ndarray: result = [result] elif type(result) == list: if type(result[0]) == np.ndarray: return result else: result = [np.array(result)] else: result = [np.array([result])] return result
def save_array(data, parameters): global need_parameters global time_stamp global expand global on_axis global file_name global disk_array global input_shape global input_type global output_type global shape_step global hdf5_file # Once the parameters are received at the starting of the graph then they cannot be updated any more. if need_parameters: try: file_name = parameters[0] time_stamp = parameters[1] expand = parameters[2] on_axis = parameters[3] output_type = parameters[4] need_parameters = False worker_object.relic_create_parameters_df(file_name=file_name, time_stamp=time_stamp, expand=expand, on_axis=on_axis, output_type=output_type) except: return message = data[1:] # data[0] is the topic array = Socket.reconstruct_array_from_bytes_message(message) if input_shape is None: try: input_type = array.dtype if expand: input_shape = list(array.shape) shape_step = np.zeros(len(input_shape)) input_shape[on_axis] = 0 shape_step[on_axis] = array.shape[on_axis] else: input_shape = [] shape_step = [] for i, n in enumerate(array.shape): if i == on_axis: input_shape.append(0) shape_step.append(1) input_shape.append(n) shape_step.append(0) max_shape = np.copy(input_shape) max_shape = list(max_shape) max_shape[np.where(np.array(input_shape) == 0)[0][0]] = None input_shape = tuple(input_shape) max_shape = tuple(max_shape) if output_type == 'Same': output_type = input_type else: output_type = np.dtype(output_type) if time_stamp: add_timestamp_to_filename() hdf5_file = h5py.File(file_name, 'w') disk_array = hdf5_file.create_dataset('data', shape=input_shape, maxshape=max_shape, dtype=output_type, chunks=True) if not expand: array = np.expand_dims(array, on_axis) add_data_to_array(array) except Exception as e: print(e) else: try: if not expand: array = np.expand_dims(array, on_axis) add_data_to_array(array) except Exception as e: print(e)