class DataAdder(QtCore.QObject):
data_updated = QtCore.Signal(object, dict)
def __init__(self, parent=None):
super().__init__(parent)
self.current_data = None
self.current_struct = {}
self.new_data_dict = {}
def set_data(self, current_data, current_struct, new_data_dict):
self.current_data = current_data
self.current_struct = current_struct
self.new_data_dict = new_data_dict
def run(self):
new_data_frames = dict_to_data_frames(self.new_data_dict)
data_struct = self.current_struct
data = {}
for data_frame in new_data_frames:
col_name = list(data_frame.columns)[0]
if self.current_data == {}:
data[col_name] = data_frame
data_struct[col_name] = get_data_structure(data_frame)
elif col_name in self.current_data:
data[col_name] = append_new_data(self.current_data[col_name], data_frame)
data_struct[col_name] = get_data_structure(data[col_name])
self.data_updated.emit(data, data_struct)
class DataStructure(QtWidgets.QTreeWidget):
data_updated = QtCore.Signal(dict)
def __init__(self, parent=None):
super().__init__(parent)
self.setColumnCount(2)
self.setHeaderLabels(['Array', 'Properties'])
self.setSelectionMode(QtWidgets.QTreeWidget.SingleSelection)
@QtCore.Slot(dict)
def update(self, structure):
for key, val in structure.items():
items = self.findItems(key, QtCore.Qt.MatchExactly)
if len(items) == 0:
# add a new option to the structure widget
item = QtWidgets.QTreeWidgetItem([key, '{} points'.format(val['nValues'])])
self.addTopLevelItem(item)
else:
item = items[0]
item.setText(1, '{} points'.format(val['nValues']))
current_selection = self.selectedItems()
if len(current_selection) == 0:
item = self.topLevelItem(0)
if item:
item.setSelected(True)
class DataReceiver(QtCore.QObject):
send_info = QtCore.Signal(str)
send_data = QtCore.Signal(dict)
def __init__(self):
super().__init__()
context = zmq.Context()
port = config['network']['port']
addr = config['network']['addr']
self.socket = context.socket(zmq.PULL)
self.socket.bind(f"tcp://{addr}:{port}")
self.running = True
@QtCore.Slot()
def loop(self):
self.send_info.emit("Listening...")
while self.running:
data_bytes = self.socket.recv()
data = json.loads(data_bytes.decode(encoding='UTF-8'))
if 'id' in data.keys():
# a proper data set requires an 'id'
data_id = data['id']
self.send_info.emit(f'Received data for dataset: {data_id}')
self.send_data.emit(data)
LOGGER.debug(f'\n\t DataReceiver received: {data} \n')
elif 'ping' in data.keys():
# so this doesn't look like an error
# when checking if the server is running
self.send_info.emit(f'Received ping.')
else:
self.send_info.emit(
f'Received invalid message '
f'(expected DataDict or ping):\n{data}'
)
class DataWindow(QtWidgets.QMainWindow):
data_added = QtCore.Signal(dict)
data_activated = QtCore.Signal(dict)
windowClosed = QtCore.Signal(str)
def __init__(self, data_id, parent=None):
super().__init__(parent)
self.data_id = data_id
search_str = 'run ID = '
idx = self.data_id.find(search_str) + len(search_str)
run_id = int(self.data_id[idx:].strip())
self.setWindowTitle(f"{get_app_title()} (#{run_id})")
self.active_dataset = None
self.data = {}
self.data_struct = {}
self.adding_queue = {}
self.plot_data_pending = False
self.current_plot_choice_info = None
# plot settings
set_matplotlib_defaults()
# data chosing widgets
self.structure_widget = DataStructure()
self.plot_choice = PlotChoice()
chooser_layout = QtWidgets.QVBoxLayout()
chooser_layout.addWidget(self.structure_widget)
chooser_layout.addWidget(self.plot_choice)
# plot control widgets
self.plot = MPLPlot(width=5, height=4)
plot_layout = QtWidgets.QVBoxLayout()
plot_layout.addWidget(self.plot)
plot_layout.addWidget(NavBar(self.plot, self))
# Main layout
self.frame = QtWidgets.QFrame()
main_layout = QtWidgets.QHBoxLayout(self.frame)
main_layout.addLayout(chooser_layout)
main_layout.addLayout(plot_layout)
# data processing threads
self.data_adder = DataAdder()
self.data_adder_thread = QtCore.QThread()
self.data_adder.moveToThread(self.data_adder_thread)
self.data_adder.data_updated.connect(self.data_from_adder)
self.data_adder.data_updated.connect(self.data_adder_thread.quit)
self.data_adder_thread.started.connect(self.data_adder.run)
self.plot_data = PlotData()
self.plot_data_thread = QtCore.QThread()
self.plot_data.moveToThread(self.plot_data_thread)
self.plot_data.data_processed.connect(self.update_plot)
self.plot_data.data_processed.connect(self.plot_data_thread.quit)
self.plot_data_thread.started.connect(self.plot_data.process_data)
# signals/slots for data selection etc.
self.data_added.connect(self.structure_widget.update)
self.data_added.connect(self.update_plot_data)
self.structure_widget.itemSelectionChanged.connect(self.activate_data)
self.data_activated.connect(self.plot_choice.set_options)
self.plot_choice.choice_updated.connect(self.update_plot_data)
# activate window
self.frame.setFocus()
self.setCentralWidget(self.frame)
self.activateWindow()
@QtCore.Slot()
def activate_data(self):
item = self.structure_widget.selectedItems()[0]
self.active_dataset = item.text(0)
self.data_activated.emit(self.data_struct[self.active_dataset])
@QtCore.Slot()
def update_plot_data(self):
if self.plot_data_thread.isRunning():
self.plot_data_pending = True
else:
self.current_plot_choice_info = self.plot_choice.choice_info
self.plot_data_pending = False
self.plot_data.set_data(
self.data[self.active_dataset],
self.current_plot_choice_info
)
self.plot_data_thread.start()
def _plot_1d_line(self, x, data):
marker = '.'
marker_size = 4
marker_color = 'k'
self.plot.axes.yaxis.set_major_formatter(mpl_formatter())
x = x.flatten() # assume this is cool
if (len(x) == data.shape[0]) or (len(x) == len(data)):
self.plot.axes.plot(
x, data,
marker=marker,
markerfacecolor=marker_color,
markeredgecolor=marker_color,
markersize=marker_size,
)
elif len(x) == data.shape[1]:
self.plot.axes.plot(
x,
data.transpose(),
marker=marker,
markerfacecolor=marker_color,
markeredgecolor=marker_color,
markersize=marker_size,
)
else:
raise ValueError('Cannot find a sensible shape for _plot_1D_line')
try:
xmin, xmax = get_axis_lims(x)
self.plot.axes.set_xlim(xmin, xmax)
except Exception as e:
LOGGER.debug(e)
self.plot.axes.set_xlabel(self.current_plot_choice_info['xAxis']['name'])
self.plot.axes.set_ylabel(self.active_dataset)
def _plot_1d_scatter(self, x, data):
self.plot.axes.yaxis.set_major_formatter(mpl_formatter())
x = x.flatten() # assume this is cool
if (len(x) == data.shape[0]) or (len(x) == len(data)):
self.plot.axes.scatter(x, data)
elif len(x) == data.shape[1]:
self.plot.axes.scatter(x, data.transpose())
else:
raise ValueError('Cannot find a sensible shape for _plot_1D_scatter')
try:
xmin, xmax = get_axis_lims(x)
self.plot.axes.set_xlim(xmin, xmax)
except Exception as e:
LOGGER.debug(e)
try:
ymin, ymax = get_axis_lims(data)
self.plot.axes.set_ylim(ymin, ymax)
except Exception as e:
LOGGER.debug(e)
self.plot.axes.set_xlabel(self.current_plot_choice_info['xAxis']['name'])
self.plot.axes.set_ylabel(self.active_dataset)
def _plot_2d_pcolor(self, x, y, data):
x_grid, y_grid = make_pcolor_grid(x, y)
if (
self.current_plot_choice_info['xAxis']['idx'] <
self.current_plot_choice_info['yAxis']['idx']
):
img = self.plot.axes.pcolormesh(x_grid, y_grid, data.transpose())
else:
img = self.plot.axes.pcolormesh(x_grid, y_grid, data)
self.plot.axes.set_xlabel(self.current_plot_choice_info['xAxis']['name'])
self.plot.axes.set_ylabel(self.current_plot_choice_info['yAxis']['name'])
cbar = self.plot.fig.colorbar(
img,
format=mpl_formatter(),
)
cbar.set_label(self.active_dataset)
def _plot_2d_scatter(self, x, y, data):
img = self.plot.axes.scatter(x, y, c=data)
try:
xmin, xmax = get_axis_lims(x)
self.plot.axes.set_xlim(xmin, xmax)
ymin, ymax = get_axis_lims(y)
self.plot.axes.set_ylim(ymin, ymax)
except Exception as e:
LOGGER.debug(e)
self.plot.axes.set_xlabel(self.current_plot_choice_info['xAxis']['name'])
self.plot.axes.set_ylabel(self.current_plot_choice_info['yAxis']['name'])
cbar = self.plot.fig.colorbar(
img,
format=mpl_formatter(),
)
cbar.set_label(self.active_dataset)
@QtCore.Slot(object, object, object, bool)
def update_plot(self, data, x_array, y_array, grid_found):
self.plot.clear_figure()
try:
pdims = get_plot_dims(data, x_array, y_array)
if pdims == 0:
raise ValueError('No data sent to DataWindow.update_plot')
if grid_found:
if pdims == 1:
self._plot_1d_line(x_array, data)
elif pdims == 2:
self._plot_2d_pcolor(x_array, y_array, data)
else:
if pdims == 1:
self._plot_1d_scatter(x_array, data)
elif pdims == 2:
self._plot_2d_scatter(x_array, y_array, data)
self.plot.axes.set_title(f"{self.data_id}", size='x-small')
self.plot.draw()
except Exception as e:
LOGGER.debug('Could not plot selected data')
LOGGER.debug(f'Exception raised: {e}')
if self.plot_data_pending:
self.update_plot_data()
def add_data(self, data_dict):
"""
Here we receive new data from the listener.
We'll use a separate thread for processing and combining (numerics might be costly).
If the thread is already running, we'll put the new data into a queue that will
be resolved during the next call of add_data (i.e, queue will grow until current
adding thread is done.)
"""
data_dict = data_dict.get('datasets', {})
if self.data_adder_thread.isRunning():
if self.adding_queue == {}:
self.adding_queue = data_dict
else:
self.adding_queue = combine_dicts(self.adding_queue, data_dict)
else:
if self.adding_queue != {}:
data_dict = combine_dicts(self.adding_queue, data_dict)
if data_dict != {}:
# move data to data_adder obj and start data_adder_thread
self.data_adder.set_data(self.data, self.data_struct, data_dict)
self.data_adder_thread.start()
self.adding_queue = {}
@QtCore.Slot(object, dict)
def data_from_adder(self, data, data_struct):
self.data = data
self.data_struct = data_struct
self.data_added.emit(self.data_struct)
# clean-up
def closeEvent(self, event):
LOGGER.debug(f'close {self.data_id}. event: {event}')
self.windowClosed.emit(self.data_id)
class PlotData(QtCore.QObject):
data_processed = QtCore.Signal(object, object, object, bool)
def set_data(self, data_frame, choice_info):
self.df = data_frame
self.choice_info = choice_info
def process_data(self):
try:
xarr = data_frame_to_xarray(self.df)
data = xarr.values[:]
if data.size != 0:
filled_elements = np.isfinite(data).sum()
total_elements = data.size
if filled_elements/total_elements < 0.05:
raise ValueError('grid is too sparse')
data_shape = list(data.shape)
exclude = [
self.choice_info['xAxis']['idx'],
self.choice_info['yAxis']['idx'],
]
squeeze_dims = tuple(
i for i in range(len(data_shape)) if (i not in exclude) and (data_shape[i] == 1)
)
plot_data = data.squeeze(squeeze_dims)
plot_data = np.ma.masked_where(np.isnan(plot_data), plot_data)
if plot_data.size < 1:
LOGGER.debug('Data has size 0')
return
if self.choice_info['xAxis']['idx'] > -1:
x_array = xarr.coords[self.choice_info['xAxis']['name']].values
else:
x_array = None
if self.choice_info['yAxis']['idx'] > -1:
y_array = xarr.coords[self.choice_info['yAxis']['name']].values
else:
y_array = None
if self.choice_info['subtractAverage']:
# This feature is only for 2D data
if x_array is not None and y_array is not None:
# x axis, horizontal one - axis 0
# y axis, vertical one - axis 1
if self.choice_info['subtractAverage'] == 'byRow':
# rows / x axis / horizontal axis
row_means = plot_data.mean(0)
row_means_matrix = row_means[np.newaxis, :]
plot_data = plot_data - row_means_matrix
elif self.choice_info['subtractAverage'] == 'byColumn':
# columns / y axis / vertical one
col_means = plot_data.mean(1)
col_means_matrix = col_means[:, np.newaxis]
plot_data = plot_data - col_means_matrix
self.data_processed.emit(plot_data, x_array, y_array, True)
return
except (ValueError, IndexError):
LOGGER.debug('PlotData.process_data: No grid for the data.')
LOGGER.debug('Fall back to scatter plot')
if self.choice_info['xAxis']['idx'] > -1:
x_label = self.choice_info['xAxis']['name']
x_array = self.df.index.get_level_values(x_label).values
else:
x_label = None
x_array = None
if self.choice_info['yAxis']['idx'] > -1:
y_label = self.choice_info['yAxis']['name']
y_array = self.df.index.get_level_values(y_label).values
else:
y_label = None
y_array = None
plot_data = self.df.values.flatten()
self.data_processed.emit(plot_data, x_array, y_array, False)
return
class PlotChoice(QtWidgets.QWidget):
choice_updated = QtCore.Signal(object)
def __init__(self, parent=None):
super().__init__(parent)
self.x_selection = QtWidgets.QComboBox()
self.y_selection = QtWidgets.QComboBox()
axis_choice_box = QtWidgets.QGroupBox('Plot axes')
axis_choice_layout = QtWidgets.QFormLayout()
axis_choice_layout.addRow(QtWidgets.QLabel('x axis'), self.x_selection)
axis_choice_layout.addRow(QtWidgets.QLabel('y axis'), self.y_selection)
axis_choice_box.setLayout(axis_choice_layout)
self.subtract_avg_box = QtWidgets.QGroupBox('Subtract average')
self.subtract_avg_button_group = QtWidgets.QButtonGroup()
self.subtract_col_avg_button = QtWidgets.QRadioButton('From each column (vertical axis)')
self.subtract_avg_button_group.addButton(self.subtract_col_avg_button, 0)
self.subtract_row_avg_button = QtWidgets.QRadioButton('From each row (horizontal axis)')
self.subtract_avg_button_group.addButton(self.subtract_row_avg_button, 1)
self.subtract_avg_none_button = QtWidgets.QRadioButton('None')
self.subtract_avg_button_group.addButton(self.subtract_avg_none_button, 2)
self.subtract_avg_none_button.setChecked(True)
self.subtract_avg_layout = QtWidgets.QFormLayout()
self.subtract_avg_layout.addRow(self.subtract_col_avg_button)
self.subtract_avg_layout.addRow(self.subtract_row_avg_button)
self.subtract_avg_layout.addRow(self.subtract_avg_none_button)
self.subtract_avg_box.setLayout(self.subtract_avg_layout)
main_layout = QtWidgets.QVBoxLayout(self)
main_layout.addWidget(axis_choice_box)
main_layout.addWidget(self.subtract_avg_box)
self.emit_choice_update = False
self.x_selection.currentTextChanged.connect(self.x_selected)
self.y_selection.currentTextChanged.connect(self.y_selected)
self.subtract_avg_button_group.buttonClicked.connect(self.subtract_average_changed)
self.empty_selection_name = '<None>'
self.axes_names = []
self.choice_info = {}
@QtCore.Slot(str)
def x_selected(self, val):
self.update_options(self.x_selection, val)
@QtCore.Slot(str)
def y_selected(self, val):
self.update_options(self.y_selection, val)
@QtCore.Slot(QtWidgets.QAbstractButton)
def subtract_average_changed(self, button):
self.update_options(None, None)
def _axis_in_use(self, name):
# for opt in self.avgSelection, self.x_selection, self.y_selection:
for opt in self.x_selection, self.y_selection:
if name == opt.currentText():
return True
return False
def update_options(self, changed_option, new_val):
"""
After changing the role of a data axis manually, we need to make
sure this axis isn't used anywhere else.
"""
for opt in self.x_selection, self.y_selection:
if opt != changed_option and opt.currentText() == new_val:
opt.setCurrentIndex(0)
subtract_average = None
if self.subtract_row_avg_button.isChecked():
subtract_average = 'byRow'
elif self.subtract_col_avg_button.isChecked():
subtract_average = 'byColumn'
self.choice_info = {
'xAxis' : {
'idx' : self.x_selection.currentIndex() - 1,
'name' : self.x_selection.currentText(),
},
'yAxis' : {
'idx' : self.y_selection.currentIndex() - 1,
'name' : self.y_selection.currentText(),
},
'subtractAverage' : subtract_average,
}
if self.emit_choice_update:
self.choice_updated.emit(self.choice_info)
@QtCore.Slot(dict)
def set_options(self, data_struct):
"""
Populates the data choice widgets initially.
"""
self.emit_choice_update = False
self.axes_names = list(data_struct['axes'].keys())
# Need an option that indicates that the choice is 'empty'
while self.empty_selection_name in self.axes_names:
self.empty_selection_name = '<' + self.empty_selection_name + '>'
self.axes_names.insert(0, self.empty_selection_name)
# add all options
for opt in self.x_selection, self.y_selection:
opt.clear()
opt.addItems(self.axes_names)
# see which options remain for x and y, apply the first that work
xopts = self.axes_names.copy()
xopts.pop(0)
if len(xopts) > 0:
self.x_selection.setCurrentText(xopts[0])
if len(xopts) > 1:
self.y_selection.setCurrentText(xopts[1])
self.emit_choice_update = True
self.choice_updated.emit(self.choice_info)