def process_data(xdata, ydata):
logger.debug('Processing data')
if config.initial_y != -1:
ydata = ydata - ydata[0] + config.initial_y
# Align at time 0 if option selected
if config.align and config.y_alignment == -1:
xdata = xdata - xdata[0]
if config.y_alignment != -1:
xdata = align_to_y(xdata, ydata, config.y_alignment)
# remove outliers
if (config.remove_above is not None or config.remove_below is not None
or config.auto_remove):
xdata, ydata = remove_outliers(xdata, ydata, config.remove_below,
config.remove_above, config.auto_remove,
config.outlier_threshold)
# Smooth the data
if (config.smooth):
ydata = savitzky_golay(ydata, config.sg_window_size, config.sg_order,
config.sg_deriv, config.sg_rate)
return xdata, ydata
def savitzky_golay(y, window_size, order, deriv=0, rate=1):
logger.debug('Smoothing data')
try:
window_size = np.abs(np.int(window_size))
order = np.abs(np.int(order))
except ValueError:
raise ValueError("window_size and order have to be of type int")
if window_size % 2 != 1 or window_size < 1:
raise TypeError("window_size size must be a positive odd number")
if window_size < order + 2:
raise TypeError("window_size is too small for the polynomials order")
order_range = range(order + 1)
half_window = (window_size - 1) // 2
# precompute coefficients
b = np.mat([[k**i for i in order_range]
for k in range(-half_window, half_window + 1)])
# On windows the first call to linalg gives nans for some reason
m = np.linalg.pinv(b).A[deriv] * rate**deriv * factorial(deriv)
if np.isnan(m).any():
m = np.linalg.pinv(b).A[deriv] * rate**deriv * factorial(deriv)
# pad the signal at the extremes with
# values taken from the signal itself
firstvals = y[0] - np.abs(y[1:half_window + 1][::-1] - y[0])
lastvals = y[-1] + np.abs(y[-half_window - 1:-1][::-1] - y[-1])
y = np.concatenate((firstvals, y, lastvals))
y = np.convolve(m[::-1], y, mode='valid')
return y
def time_average(xdata, ydata, window, show_err=False):
logger.debug('Averaging data over time window of %i' % window)
new_xdata = np.array([])
new_ydata = np.array([])
new_yerr = np.array([])
w_i = 1
i = 0
while (i < len(xdata)):
data_x = np.array([])
data_y = np.array([])
while (i < len(xdata) and xdata[i] < w_i * window):
data_x = np.append(data_x, xdata[i])
data_y = np.append(data_y, ydata[i])
i = i + 1
if (data_y.size == 0):
w_i = w_i + 1
continue
mean_x = np.mean(data_x)
mean_y = np.mean(data_y)
std_dev = np.std(data_y, ddof=1)
if (show_err):
std_dev = std_dev / np.sqrt(data_y.size)
new_xdata = np.append(new_xdata, mean_x)
new_ydata = np.append(new_ydata, mean_y)
if (data_y.size == 1):
new_yerr = np.append(new_yerr, 0)
else:
new_yerr = np.append(new_yerr, std_dev)
w_i = w_i + 1
return new_xdata, new_ydata, new_yerr
def update_data_list(self):
logger.debug('Updating the list of data files')
self.data_list.clear()
self.yaxis_dropdown.clear()
self.legend_names.clear()
for i, data in enumerate(data_manager.get_growth_data_files()):
data_list_item = DataListItem(data.label, i, self)
self.data_list.addItem(data_list_item.item)
self.data_list.setItemWidget(data_list_item.item,
data_list_item.widget)
if data.legend:
self.legend_names.addItem(data.legend)
else:
self.legend_names.addItem(data.label)
if i > 0:
continue
contains_od = False
contains_cd = False
for sig in reversed(data.signals):
self.yaxis_dropdown.addItem(sig.name)
if sig.name == 'OD':
contains_od = True
if sig.name == 'CD':
contains_cd = True
if contains_od and not contains_cd and data_manager.calibration is not None:
self.yaxis_dropdown.addItem('CD')
def get_condition_at(self, cond_name, time):
logger.debug('Getting condition %s at time %.2f' % (cond_name, time))
values = []
for i, _ in enumerate(self.growth_data.data_files):
xdata, ydata, _ = self.get_condition_xy_data(i, cond_name)
values.append(np.interp(time, xdata, ydata))
return values
def __init__(self, name, width, height, layout=None, parent=None, tabbed=False):
super(Window, self).__init__(parent)
self.title = name
self.width = width*config.wr
self.height = height*config.hr
logger.debug('Creating %s window [width:%.2f, height:%.2f]' % (
name, self.width, self.height))
self.parent = parent
self.setWindowTitle(self.title)
if tabbed:
self.tabs = QTabWidget()
else:
self.window = LayoutWidget(layout)
self.window.layout.setContentsMargins(
5*config.wr, 5*config.hr, 5*config.wr, 5*config.hr)
self.window.layout.setSpacing(5*config.wr)
if tabbed:
self.tabs.setStyleSheet(styles.tab_style)
self.setCentralWidget(self.tabs)
else:
self.setCentralWidget(self.window.widget)
self.resize(self.width, self.height)
def open_calibration_file(self):
logger.debug('Loading calibration curve from file')
self.calibration_file.clear()
calib_file_name = get_file_names()
self.calibration_file.setText(calib_file_name[0])
data_manager.calibration = read_calibration(calib_file_name[0])
self.update_data_list()
def plot(self, plot_config=None):
logger.debug('Creating correlation plot')
self.axes.clear()
if plot_config is None:
return
self.scatter = self.axes.scatter(
plot_config.x_data, plot_config.y_data, alpha=0)
self.errbar = self.axes.errorbar(
plot_config.x_data, plot_config.y_data, plot_config.y_error, plot_config.x_error, '.')
if plot_config is not None:
self.axes.set_title(plot_config.title)
self.axes.set_xlabel(plot_config.x_title)
self.axes.set_ylabel(plot_config.y_title)
bounding_box = dict(boxstyle="round", ec=(
1., 0.5, 0.5), fc=(1., 0.8, 0.8))
if plot_config.correlation_coeff is not None:
text = ('$\\rho$ = %.*f' %
(config.sig_figs, plot_config.correlation_coeff))
self.axes.text(0.25, 0.95, text,
transform=self.axes.transAxes,
bbox=bounding_box, picker=True)
self.label_annotation = self.axes.annotate('',
xy=(0, 0),
xytext=(0.2, 0.2),
textcoords='axes fraction',
bbox=dict(
boxstyle="round", fc="w"),
arrowprops=dict(arrowstyle="->"))
self.label_annotation.set_visible(False)
def update_annotation(ind):
label_pos = self.scatter.get_offsets()[ind["ind"][0]]
self.label_annotation.xy = label_pos
label_text = plot_config.labels[ind["ind"][0]]
self.label_annotation.set_text(label_text)
def onhover(event):
vis = self.label_annotation.get_visible()
cont = None
if event.inaxes == self.axes:
cont, ind = self.scatter.contains(event)
if cont:
update_annotation(ind)
self.label_annotation.set_visible(True)
self.draw_idle()
else:
if vis:
self.label_annotation.set_visible(False)
self.draw_idle()
if len(plot_config.labels) > 0:
self.mpl_connect('motion_notify_event', onhover)
self.draw()
return
def export(self):
path = self.test_path
if self.test_path == 'none':
path = get_save_directory_name()
logger.info('Exporting files to %s' % path)
for data in data_manager.get_growth_data_files():
filename = data.label + '.csv'
if self.rename.isChecked():
filename = path + '/' + data.profile + '_ada.csv'
else:
filename = path + '/' + \
filename.split('/')[-1].split('.')[0] + '_ada.csv'
logger.debug('Exporting file %s' % filename)
# Get the condition data if that option is checked
conditions = None
if self.conditions.isChecked():
for cond_data in data_manager.get_condition_data_files():
if data.reactor != cond_data.reactor:
continue
if data.date != cond_data.date:
continue
if data.time != cond_data.time:
continue
conditions = cond_data
with open(filename, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
name_header = [
'Name', data.label, 'Title', data.title, 'Reactor',
data.reactor, 'Profile', data.profile
]
writer.writerow(name_header)
date_header = ['Date', data.date, 'Time', data.time]
writer.writerow(date_header)
measurement_header = [
data.xaxis.name + ' [' + data.xaxis.unit + ']'
]
for sig in data.signals:
measurement_header.append(sig.name + ' [' + sig.unit + ']')
if conditions is not None:
measurement_header.append('Conditions')
for sig in conditions.signals:
measurement_header.append(sig.name + ' [' + sig.unit +
']')
writer.writerow(measurement_header)
for i, xdat in enumerate(data.xaxis.data):
row = [xdat]
for sig in data.signals:
row.append(sig.data[i])
# Find closest signal time
if conditions is not None:
row.append('')
cond_ind = (np.abs(conditions.xaxis.data -
xdat)).argmin()
for sig in conditions.signals:
row.append(sig.data[cond_ind])
writer.writerow(row)
self.close()
def get_replicate_gradients(self, i, signal_name, grad_from, grad_to):
logger.debug('Getting gradient of %s from %.2f to %.2f' %
(signal_name, grad_from, grad_to))
gradients = []
xdatas, ydatas = self.get_replicate_xy_data(i, signal_name)
for rep_i, xdata in enumerate(xdatas):
gradients.append(calculate_gradient(xdata, ydatas[rep_i], grad_from, grad_to))
return gradients
def fit_curve(self):
if not config.do_fit:
logger.debug('Opening fit window')
self.fit = FitWindow(self)
self.fit.show()
else:
config.do_fit = False
self.update_plot()
def remove_condition_item(self):
row = self.get_condition_row()
logger.debug('Removing condition data list item %i' % row)
for i, _ in enumerate(data_manager.get_condition_data_files()):
if i != row:
continue
data_manager.condition_data.delete_data(i)
self.update_condition_data_list()
def get_gradients(self, signal_name, grad_from, grad_to):
logger.debug('Getting gradient of %s from %.2f to %.2f' %
(signal_name, grad_from, grad_to))
gradients = []
for i, _ in enumerate(self.growth_data.data_files):
xdata, ydata, _ = self.get_xy_data(i, signal_name)
gradients.append(calculate_gradient(xdata, ydata, grad_from, grad_to))
return gradients
def get_replicate_time_to(self, i, signal_name, time_to):
logger.debug('Getting the time to reach %s of %.2f' %
(signal_name, time_to))
times = []
xdatas, ydatas = self.get_replicate_xy_data(i, signal_name)
for rep_i, xdata in enumerate(xdatas):
times.append(calculate_time_to(xdata, ydatas[rep_i], time_to))
return times
def fill_list(self, file_list, display_list):
logger.debug('Creating list of files to be loaded')
display_list.clear()
for file_name in file_list:
list_item = DelListItem(file_name.split('/')[-1])
list_item.button.clicked.connect(
lambda: self.remove_item(file_list, display_list))
display_list.addItem(list_item.item)
display_list.setItemWidget(list_item.item, list_item.widget)
def remove_condition_replicate(self, index):
row = self.get_condition_row()
logger.debug('Removing replicate %i from condition list item %i' %
(index, row))
for i, _ in enumerate(data_manager.get_condition_data_files()):
if i != row:
continue
data_manager.condition_data.delete_replicate(i, index)
self.update_condition_data_list()
def on_context_menu(self, point):
# show context menu
action = self.clear_menu.exec_(self.data_button.mapToGlobal(point))
if action == self.clear_action:
logger.debug('Clearing all data')
data_manager.clear()
self.remove_calibration_file()
self.update_data_list()
self.update_condition_data_list()
def get_time_to(self, signal_name, time_to):
logger.debug('Getting the time to reach %s of %.2f' %
(signal_name, time_to))
times = []
for i, _ in enumerate(self.growth_data.data_files):
xdata, ydata, _ = self.get_xy_data(i, signal_name)
times.append(calculate_time_to(xdata, ydata, time_to))
return times
def align_to_y(xdata, ydata, y_alignment):
logger.debug('Aligning data to y = %.2f' % y_alignment)
# Find the first y index greater than the alignment point
index = 0
for i, y in enumerate(ydata):
if y >= y_alignment:
index = i
break
xdata = xdata - xdata[index]
return xdata
def remove_item(self, file_list, display_list):
widget = self.sender()
gp = widget.mapToGlobal(QPoint())
lp = display_list.viewport().mapFromGlobal(gp)
row = display_list.row(display_list.itemAt(lp))
logger.debug('Removing item %i from data list' % row)
for i, _ in enumerate(file_list):
if i != row:
continue
del file_list[i]
self.fill_list(file_list, display_list)
def download_template(self):
logger.debug('Downloading ADA data template')
template = [
'Name,,Title,,Reactor,,Profile,\n',
'Date,2020-01-15,Time,18:18:18\n', 'Time [hr],OD [],Conditions\n'
]
file_name = get_save_file_name()
file_name = file_name.split('.')[0] + '.csv'
with open(file_name, 'w', newline='') as csvfile:
for row in template:
csvfile.write(row)
def openFileNamesDialog(self):
logger.debug('Creating window for getting file names for opening')
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
files, _ = QFileDialog.getOpenFileNames(
self,
"Open File(s)",
"",
"All Files (*);;Text Files (*.txt)",
options=options)
self.file_names = files
def __init__(self):
super().__init__()
self.title = 'Algal Data Analyser'
# Default dimensions
self.left = 10 * config.wr
self.top = 60 * config.wr
self.width = 960 * config.wr
self.height = 600 * config.wr
logger.debug(
'Creating main window [left:%.2f, top:%.2f, width:%.2f, height:%.2f]'
% (self.left, self.top, self.width, self.height))
self.setStyleSheet(styles.main_background)
self._createMenuBar()
self.initUI()
def apply_changes(self):
logger.debug('Trying to set colour %s and style %s' %
(self.line_colour.text(), self.line_style.currentText()))
try:
if (is_color_like(self.line_colour.text())):
self.data.style["color"] = self.line_colour.text()
self.data.style["linestyle"] = self.line_style.currentText()
self.data.style["marker"] = config.marker_style_options[
self.marker_style.currentText()]
self.parent.set_plot_styles()
self.parent.draw()
self.close()
except Exception as e:
logger.exception(e)
logger.warning('Unable change style, skipping')
pass
def get_all_fit_params(self, signal_name, fit_name, fit_from, fit_to, fit_param):
logger.debug('Fitting %s with %s from %.2f to %.2f and recording %s' % (
signal_name, fit_name, fit_from, fit_to, fit_param))
values = []
errors = []
for i, _ in enumerate(self.growth_data.data_files):
fit_result, covm = self.get_fit(
i, signal_name, fit_name, fit_from, fit_to)
param_errors = np.sqrt(np.diag(covm))
model = get_model(fit_name)
for i, param in enumerate(model.params):
if param == fit_param:
values.append(fit_result[i])
errors.append(param_errors[i])
return values, errors
def saveFileDialog(self):
logger.debug('Creating window for creating file name for saving')
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
self.file_name, _ = QFileDialog.getSaveFileName(
self,
"Save File",
"",
"All Files (*);;Text Files (*.txt)",
options=options)
if self.file_name and self.save_fig:
if (self.file_name == ''):
self.fig.savefig('graph.png', dpi=1200)
elif (self.file_name.find('.') == -1):
self.fig.savefig(self.file_name + '.png')
else:
self.fig.savefig(self.file_name)
def update_condition_data_list(self):
logger.debug('Updating the list of condition data files')
self.condition_data_list.clear()
self.condition_yaxis_dropdown.clear()
self.condition_legend_names.clear()
for i, data in enumerate(data_manager.get_condition_data_files()):
data_list_item = ConditionListItem(data.label, i, self)
self.condition_data_list.addItem(data_list_item.item)
self.condition_data_list.setItemWidget(data_list_item.item,
data_list_item.widget)
if data.legend:
self.condition_legend_names.addItem(data.legend)
else:
self.condition_legend_names.addItem(data.label)
if i > 0:
continue
for sig in data.signals:
self.condition_yaxis_dropdown.addItem(sig.name)
def __init__(self, purpose='open', fig=None):
super().__init__()
if purpose == 'open':
self.title = 'Open Files'
elif purpose == 'save':
self.title = 'Save File'
elif purpose == 'directory':
self.title = 'Choose Directory'
self.purpose = purpose
if fig:
self.fig = fig
self.save_fig = True
else:
self.save_fig = False
self.width = 960 * config.wr
self.height = 600 * config.hr
logger.debug('Creating file handler window [width:%.2f, height:%.2f]' %
(self.width, self.height))
self.initUI()
def load(self):
logger.debug('Loading files into ADA')
file_type = self.file_type.currentText()
for file_name in self.files:
if file_type == 'Algem Pro' and file_name.endswith('.txt'):
self.load_algem_pro(file_name)
elif file_type == 'Algem HT24' and file_name.endswith('.txt'):
self.load_algem_ht24_txt(file_name)
elif file_type == 'Algem HT24' and file_name.endswith('.csv'):
self.load_algem_ht24(file_name)
elif file_type == 'IP' and file_name.endswith('.csv'):
self.load_ip(file_name)
elif file_type == 'PSI' and file_name.endswith('.ods'):
self.load_psi(file_name)
elif file_type == 'ADA' and file_name.endswith('.csv'):
self.load_ada(file_name)
elif file_type == 'MicrobeMeter' and file_name.endswith('.tsv'):
self.load_microbemeter(file_name)
else:
raise RuntimeError("File %s has the wrong extension" %
(file_name))
if len(self.condition_files) > 0:
# Set downsampling if option selected
downsample = -1
if isint(self.downsample.text()):
downsample = int(self.downsample.text())
# Load in optional conditions data for algem machines
for file_name in self.condition_files:
if file_type == 'Algem Pro' and file_name.endswith('.txt'):
self.load_algem_pro_conditions(file_name, downsample)
elif file_type == 'Algem HT24' and file_name.endswith('.csv'):
self.load_algem_ht24_conditions(file_name, downsample)
else:
raise RuntimeError("File %s has the wrong extension" %
(file_name))
# Update the data lists in the main window
self.parent.update_data_list()
self.parent.update_condition_data_list()
self.close()
def average_data(xdatas, ydatas, show_err=False):
logger.debug('Averaging data')
new_xdata = np.array([])
new_ydata = np.array([])
new_yerr = np.array([])
if len(xdatas) <= 1:
return xdatas[0], ydatas[0], new_yerr
for i, x_i in enumerate(xdatas[0]):
ys = np.array([ydatas[0][i]])
for j in range(1, len(xdatas), 1):
# Interpolate between points to do average
ys = np.append(ys, np.interp(x_i, xdatas[j], ydatas[j]))
mean = np.mean(ys)
std_dev = np.std(ys, ddof=1)
if (show_err):
std_dev = std_dev / np.sqrt(ys.size)
new_xdata = np.append(new_xdata, x_i)
new_ydata = np.append(new_ydata, mean)
new_yerr = np.append(new_yerr, std_dev)
return new_xdata, new_ydata, new_yerr
