def on_receive_data_from_worker(self, msg):
"""
The callback that runs every time link is received from the worker_exec process. It takes the link and passes it
onto the link forwarder
:param msg: The link packet (carrying the actual link (np array))
:return:
"""
# A specific worker with multiple outputs should send from its infinite loop a message with multiple parts
# (using multiple send_array(data, flags=zmq.SNDMORE) commands). For an example see how the transform_worker
# sends data to the com from its data_callback function
# TODO The bellow will not work for multiple outputs. I have to find out how many times a callback is called
# when data are send with SNDMORE flag !!!
ignoring_outputs = [False] * len(self.outputs)
new_message_data = []
if len(self.outputs) > 1:
for i in range(len(self.outputs)):
array_data = Socket.reconstruct_array_from_bytes_message(msg[i])
new_message_data.append(array_data)
if type(array_data[0]) == np.str_:
if array_data[0] == ct.IGNORE:
ignoring_outputs[i] = True
else:
array_data = Socket.reconstruct_array_from_bytes_message(msg)
new_message_data.append(array_data)
if type(array_data[0]) == np.str_:
if array_data[0] == ct.IGNORE:
ignoring_outputs[0] = True
self.time = int(1000000 * time.perf_counter())
self.index = self.index + 1
# Publish the results. Each array in the list of arrays is published to its own sending topic
# (matched by order)
for i, st in enumerate(self.sending_topics):
for k, output in enumerate(self.outputs):
if output.replace(' ', '_') in st.split('##')[0]:
break
if ignoring_outputs[k] is False:
self.socket_pub_data.send("{}".format(st).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send("{}".format(self.index).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send("{}".format(self.time).encode('ascii'), flags=zmq.SNDMORE)
self.socket_pub_data.send_array(new_message_data[k], copy=False)
# This delay is critical to get single output to multiple inputs to work!
gu.accurate_delay(ct.DELAY_BETWEEN_SENDING_DATA_TO_NEXT_NODE_MILLISECONDS)
if self.verbose:
dt = self.time - self.previous_time
if self.index > 3:
self.average_sending_time = self.average_sending_time * (self.index - 1) / self.index + dt / self.index
print('----------')
print("Source with topic {} sending packet with data_index {} at time {}".format(self.sending_topics[i],
self.index, self.time))
print('Time Diff between packages = {}. Average package sending time = {} ms'.format(dt/1000, self.average_sending_time / 1000))
if self.logger:
self.logger.info('{} : {}'.format(self.index, datetime.now()))
self.previous_time = self.time
def visualise(msg, parameters):
global visualisation_type
global visualisation_on
global data
message = msg[1:] # data[0] is the topic
data = Socket.reconstruct_array_from_bytes_message(message)
if parameters is not None:
visualisation_on = parameters[0]
try:
if visualisation_type == 'Image':
show_image()
elif visualisation_type == 'Value':
update_dpg_gui()
elif visualisation_type == 'Single Pane Plot':
update_dpg_gui()
elif visualisation_type == 'Multi Pane Plot':
update_dpg_gui()
elif visualisation_type == 'Histogram':
pass
except Exception as e:
print(e)
return [data]
def work_function(data, parameters):
global global_var_1
global global_var_2
global global_var_3
global global_var_4
global vis
'''
# This is a 2nd way to initialise the parameters if no initialisation function is used.
global need_parameters
if need_parameters:
try:
global_var_1 = parameters[1]
global_var_2 = parameters[2]
global_var_3 = parameters[3]
global_var_1 = parameters[4]
need_parameters = True
except:
pass
else:
# Do the rest of the code in here so that it doesn't run if the parameters are not set
'''
# If any parameters need to be updated during runtime then do that here, e.g.
# Also update the visualisation parameter. This allows to turn on and off the visualisation window during
# run time
try:
global_var_1 = parameters[1]
vis.visualisation_on = parameters[0]
except:
pass
# In the case of multiple inputs the topic will tell you which input the message has come from. The topic is a
# string that is formatted as follows:
# previous_node_name##previous_node_index##previous_node_output_name -> this_none_name##this_node_index##this_node_input_name
# so you can see which input the data is coming from by looking at the ##this_node_input_name part. Also although
# the names of the inputs and outputs can have spaces, these become underscores in the names of the topics.
topic = data[0]
print(
topic
) # prints will not work if the operation is running on a different computer.
# The message is a numpy array send in two parts, a header dic (as bytes0 with the array's info and list of bytes
# that carry the array's payload.
message = data[1:]
message = Socket.reconstruct_array_from_bytes_message(message)[
0] # This is needed to reconstruct the message
# that comes in into the numpy array that it is.
# Now do stuff
print(message.shape)
# Whatever data the Node must visualise should be put in the vis.visualised_data variable
vis.visualised_data = np.random.random((100, 100))
def work_function(data, parameters):
global vis
global df
try:
vis.visualisation_on = parameters[0]
except:
pass
topic = data[0]
message = data[1:]
message = Socket.reconstruct_array_from_bytes_message(message)
#print('--- Message in Save DF: {}'.format(message))
time = datetime.now()
row = pd.DataFrame([message], columns=df.columns, index=[time])
#print('--- DF Row: {}'.format(row))
df = pd.concat([df, row], ignore_index=False)
def work_function(data, parameters):
global function
topic = data[0].decode('utf-8')
message = data[1:] # data[0] is the topic
input_data = Socket.reconstruct_array_from_bytes_message(message)
result = function(topic, input_data)
if type(result) == np.ndarray:
result = [result]
elif type(result) == list:
if type(result[0]) == np.ndarray:
return result
else:
result = [np.array(result)]
else:
result = [np.array([result])]
return result
def save_array(data, parameters):
global need_parameters
global time_stamp
global expand
global on_axis
global file_name
global disk_array
global input_shape
global input_type
global output_type
global shape_step
global hdf5_file
# Once the parameters are received at the starting of the graph then they cannot be updated any more.
if need_parameters:
try:
file_name = parameters[0]
time_stamp = parameters[1]
expand = parameters[2]
on_axis = parameters[3]
output_type = parameters[4]
need_parameters = False
worker_object.relic_create_parameters_df(file_name=file_name,
time_stamp=time_stamp,
expand=expand,
on_axis=on_axis,
output_type=output_type)
except:
return
message = data[1:] # data[0] is the topic
array = Socket.reconstruct_array_from_bytes_message(message)
if input_shape is None:
try:
input_type = array.dtype
if expand:
input_shape = list(array.shape)
shape_step = np.zeros(len(input_shape))
input_shape[on_axis] = 0
shape_step[on_axis] = array.shape[on_axis]
else:
input_shape = []
shape_step = []
for i, n in enumerate(array.shape):
if i == on_axis:
input_shape.append(0)
shape_step.append(1)
input_shape.append(n)
shape_step.append(0)
max_shape = np.copy(input_shape)
max_shape = list(max_shape)
max_shape[np.where(np.array(input_shape) == 0)[0][0]] = None
input_shape = tuple(input_shape)
max_shape = tuple(max_shape)
if output_type == 'Same':
output_type = input_type
else:
output_type = np.dtype(output_type)
if time_stamp:
add_timestamp_to_filename()
hdf5_file = h5py.File(file_name, 'w')
disk_array = hdf5_file.create_dataset('data',
shape=input_shape,
maxshape=max_shape,
dtype=output_type,
chunks=True)
if not expand:
array = np.expand_dims(array, on_axis)
add_data_to_array(array)
except Exception as e:
print(e)
else:
try:
if not expand:
array = np.expand_dims(array, on_axis)
add_data_to_array(array)
except Exception as e:
print(e)
