def process_data(xdata, ydata): logger.debug('Processing data') if config.initial_y != -1: ydata = ydata - ydata[0] + config.initial_y # Align at time 0 if option selected if config.align and config.y_alignment == -1: xdata = xdata - xdata[0] if config.y_alignment != -1: xdata = align_to_y(xdata, ydata, config.y_alignment) # remove outliers if (config.remove_above is not None or config.remove_below is not None or config.auto_remove): xdata, ydata = remove_outliers(xdata, ydata, config.remove_below, config.remove_above, config.auto_remove, config.outlier_threshold) # Smooth the data if (config.smooth): ydata = savitzky_golay(ydata, config.sg_window_size, config.sg_order, config.sg_deriv, config.sg_rate) return xdata, ydata
def savitzky_golay(y, window_size, order, deriv=0, rate=1): logger.debug('Smoothing data') try: window_size = np.abs(np.int(window_size)) order = np.abs(np.int(order)) except ValueError: raise ValueError("window_size and order have to be of type int") if window_size % 2 != 1 or window_size < 1: raise TypeError("window_size size must be a positive odd number") if window_size < order + 2: raise TypeError("window_size is too small for the polynomials order") order_range = range(order + 1) half_window = (window_size - 1) // 2 # precompute coefficients b = np.mat([[k**i for i in order_range] for k in range(-half_window, half_window + 1)]) # On windows the first call to linalg gives nans for some reason m = np.linalg.pinv(b).A[deriv] * rate**deriv * factorial(deriv) if np.isnan(m).any(): m = np.linalg.pinv(b).A[deriv] * rate**deriv * factorial(deriv) # pad the signal at the extremes with # values taken from the signal itself firstvals = y[0] - np.abs(y[1:half_window + 1][::-1] - y[0]) lastvals = y[-1] + np.abs(y[-half_window - 1:-1][::-1] - y[-1]) y = np.concatenate((firstvals, y, lastvals)) y = np.convolve(m[::-1], y, mode='valid') return y
def time_average(xdata, ydata, window, show_err=False): logger.debug('Averaging data over time window of %i' % window) new_xdata = np.array([]) new_ydata = np.array([]) new_yerr = np.array([]) w_i = 1 i = 0 while (i < len(xdata)): data_x = np.array([]) data_y = np.array([]) while (i < len(xdata) and xdata[i] < w_i * window): data_x = np.append(data_x, xdata[i]) data_y = np.append(data_y, ydata[i]) i = i + 1 if (data_y.size == 0): w_i = w_i + 1 continue mean_x = np.mean(data_x) mean_y = np.mean(data_y) std_dev = np.std(data_y, ddof=1) if (show_err): std_dev = std_dev / np.sqrt(data_y.size) new_xdata = np.append(new_xdata, mean_x) new_ydata = np.append(new_ydata, mean_y) if (data_y.size == 1): new_yerr = np.append(new_yerr, 0) else: new_yerr = np.append(new_yerr, std_dev) w_i = w_i + 1 return new_xdata, new_ydata, new_yerr
def update_data_list(self): logger.debug('Updating the list of data files') self.data_list.clear() self.yaxis_dropdown.clear() self.legend_names.clear() for i, data in enumerate(data_manager.get_growth_data_files()): data_list_item = DataListItem(data.label, i, self) self.data_list.addItem(data_list_item.item) self.data_list.setItemWidget(data_list_item.item, data_list_item.widget) if data.legend: self.legend_names.addItem(data.legend) else: self.legend_names.addItem(data.label) if i > 0: continue contains_od = False contains_cd = False for sig in reversed(data.signals): self.yaxis_dropdown.addItem(sig.name) if sig.name == 'OD': contains_od = True if sig.name == 'CD': contains_cd = True if contains_od and not contains_cd and data_manager.calibration is not None: self.yaxis_dropdown.addItem('CD')
def get_condition_at(self, cond_name, time): logger.debug('Getting condition %s at time %.2f' % (cond_name, time)) values = [] for i, _ in enumerate(self.growth_data.data_files): xdata, ydata, _ = self.get_condition_xy_data(i, cond_name) values.append(np.interp(time, xdata, ydata)) return values
def __init__(self, name, width, height, layout=None, parent=None, tabbed=False): super(Window, self).__init__(parent) self.title = name self.width = width*config.wr self.height = height*config.hr logger.debug('Creating %s window [width:%.2f, height:%.2f]' % ( name, self.width, self.height)) self.parent = parent self.setWindowTitle(self.title) if tabbed: self.tabs = QTabWidget() else: self.window = LayoutWidget(layout) self.window.layout.setContentsMargins( 5*config.wr, 5*config.hr, 5*config.wr, 5*config.hr) self.window.layout.setSpacing(5*config.wr) if tabbed: self.tabs.setStyleSheet(styles.tab_style) self.setCentralWidget(self.tabs) else: self.setCentralWidget(self.window.widget) self.resize(self.width, self.height)
def open_calibration_file(self): logger.debug('Loading calibration curve from file') self.calibration_file.clear() calib_file_name = get_file_names() self.calibration_file.setText(calib_file_name[0]) data_manager.calibration = read_calibration(calib_file_name[0]) self.update_data_list()
def plot(self, plot_config=None): logger.debug('Creating correlation plot') self.axes.clear() if plot_config is None: return self.scatter = self.axes.scatter( plot_config.x_data, plot_config.y_data, alpha=0) self.errbar = self.axes.errorbar( plot_config.x_data, plot_config.y_data, plot_config.y_error, plot_config.x_error, '.') if plot_config is not None: self.axes.set_title(plot_config.title) self.axes.set_xlabel(plot_config.x_title) self.axes.set_ylabel(plot_config.y_title) bounding_box = dict(boxstyle="round", ec=( 1., 0.5, 0.5), fc=(1., 0.8, 0.8)) if plot_config.correlation_coeff is not None: text = ('$\\rho$ = %.*f' % (config.sig_figs, plot_config.correlation_coeff)) self.axes.text(0.25, 0.95, text, transform=self.axes.transAxes, bbox=bounding_box, picker=True) self.label_annotation = self.axes.annotate('', xy=(0, 0), xytext=(0.2, 0.2), textcoords='axes fraction', bbox=dict( boxstyle="round", fc="w"), arrowprops=dict(arrowstyle="->")) self.label_annotation.set_visible(False) def update_annotation(ind): label_pos = self.scatter.get_offsets()[ind["ind"][0]] self.label_annotation.xy = label_pos label_text = plot_config.labels[ind["ind"][0]] self.label_annotation.set_text(label_text) def onhover(event): vis = self.label_annotation.get_visible() cont = None if event.inaxes == self.axes: cont, ind = self.scatter.contains(event) if cont: update_annotation(ind) self.label_annotation.set_visible(True) self.draw_idle() else: if vis: self.label_annotation.set_visible(False) self.draw_idle() if len(plot_config.labels) > 0: self.mpl_connect('motion_notify_event', onhover) self.draw() return
def export(self): path = self.test_path if self.test_path == 'none': path = get_save_directory_name() logger.info('Exporting files to %s' % path) for data in data_manager.get_growth_data_files(): filename = data.label + '.csv' if self.rename.isChecked(): filename = path + '/' + data.profile + '_ada.csv' else: filename = path + '/' + \ filename.split('/')[-1].split('.')[0] + '_ada.csv' logger.debug('Exporting file %s' % filename) # Get the condition data if that option is checked conditions = None if self.conditions.isChecked(): for cond_data in data_manager.get_condition_data_files(): if data.reactor != cond_data.reactor: continue if data.date != cond_data.date: continue if data.time != cond_data.time: continue conditions = cond_data with open(filename, 'w', newline='') as csvfile: writer = csv.writer(csvfile) name_header = [ 'Name', data.label, 'Title', data.title, 'Reactor', data.reactor, 'Profile', data.profile ] writer.writerow(name_header) date_header = ['Date', data.date, 'Time', data.time] writer.writerow(date_header) measurement_header = [ data.xaxis.name + ' [' + data.xaxis.unit + ']' ] for sig in data.signals: measurement_header.append(sig.name + ' [' + sig.unit + ']') if conditions is not None: measurement_header.append('Conditions') for sig in conditions.signals: measurement_header.append(sig.name + ' [' + sig.unit + ']') writer.writerow(measurement_header) for i, xdat in enumerate(data.xaxis.data): row = [xdat] for sig in data.signals: row.append(sig.data[i]) # Find closest signal time if conditions is not None: row.append('') cond_ind = (np.abs(conditions.xaxis.data - xdat)).argmin() for sig in conditions.signals: row.append(sig.data[cond_ind]) writer.writerow(row) self.close()
def get_replicate_gradients(self, i, signal_name, grad_from, grad_to): logger.debug('Getting gradient of %s from %.2f to %.2f' % (signal_name, grad_from, grad_to)) gradients = [] xdatas, ydatas = self.get_replicate_xy_data(i, signal_name) for rep_i, xdata in enumerate(xdatas): gradients.append(calculate_gradient(xdata, ydatas[rep_i], grad_from, grad_to)) return gradients
def fit_curve(self): if not config.do_fit: logger.debug('Opening fit window') self.fit = FitWindow(self) self.fit.show() else: config.do_fit = False self.update_plot()
def remove_condition_item(self): row = self.get_condition_row() logger.debug('Removing condition data list item %i' % row) for i, _ in enumerate(data_manager.get_condition_data_files()): if i != row: continue data_manager.condition_data.delete_data(i) self.update_condition_data_list()
def get_gradients(self, signal_name, grad_from, grad_to): logger.debug('Getting gradient of %s from %.2f to %.2f' % (signal_name, grad_from, grad_to)) gradients = [] for i, _ in enumerate(self.growth_data.data_files): xdata, ydata, _ = self.get_xy_data(i, signal_name) gradients.append(calculate_gradient(xdata, ydata, grad_from, grad_to)) return gradients
def get_replicate_time_to(self, i, signal_name, time_to): logger.debug('Getting the time to reach %s of %.2f' % (signal_name, time_to)) times = [] xdatas, ydatas = self.get_replicate_xy_data(i, signal_name) for rep_i, xdata in enumerate(xdatas): times.append(calculate_time_to(xdata, ydatas[rep_i], time_to)) return times
def fill_list(self, file_list, display_list): logger.debug('Creating list of files to be loaded') display_list.clear() for file_name in file_list: list_item = DelListItem(file_name.split('/')[-1]) list_item.button.clicked.connect( lambda: self.remove_item(file_list, display_list)) display_list.addItem(list_item.item) display_list.setItemWidget(list_item.item, list_item.widget)
def remove_condition_replicate(self, index): row = self.get_condition_row() logger.debug('Removing replicate %i from condition list item %i' % (index, row)) for i, _ in enumerate(data_manager.get_condition_data_files()): if i != row: continue data_manager.condition_data.delete_replicate(i, index) self.update_condition_data_list()
def on_context_menu(self, point): # show context menu action = self.clear_menu.exec_(self.data_button.mapToGlobal(point)) if action == self.clear_action: logger.debug('Clearing all data') data_manager.clear() self.remove_calibration_file() self.update_data_list() self.update_condition_data_list()
def get_time_to(self, signal_name, time_to): logger.debug('Getting the time to reach %s of %.2f' % (signal_name, time_to)) times = [] for i, _ in enumerate(self.growth_data.data_files): xdata, ydata, _ = self.get_xy_data(i, signal_name) times.append(calculate_time_to(xdata, ydata, time_to)) return times
def align_to_y(xdata, ydata, y_alignment): logger.debug('Aligning data to y = %.2f' % y_alignment) # Find the first y index greater than the alignment point index = 0 for i, y in enumerate(ydata): if y >= y_alignment: index = i break xdata = xdata - xdata[index] return xdata
def remove_item(self, file_list, display_list): widget = self.sender() gp = widget.mapToGlobal(QPoint()) lp = display_list.viewport().mapFromGlobal(gp) row = display_list.row(display_list.itemAt(lp)) logger.debug('Removing item %i from data list' % row) for i, _ in enumerate(file_list): if i != row: continue del file_list[i] self.fill_list(file_list, display_list)
def download_template(self): logger.debug('Downloading ADA data template') template = [ 'Name,,Title,,Reactor,,Profile,\n', 'Date,2020-01-15,Time,18:18:18\n', 'Time [hr],OD [],Conditions\n' ] file_name = get_save_file_name() file_name = file_name.split('.')[0] + '.csv' with open(file_name, 'w', newline='') as csvfile: for row in template: csvfile.write(row)
def openFileNamesDialog(self): logger.debug('Creating window for getting file names for opening') options = QFileDialog.Options() options |= QFileDialog.DontUseNativeDialog files, _ = QFileDialog.getOpenFileNames( self, "Open File(s)", "", "All Files (*);;Text Files (*.txt)", options=options) self.file_names = files
def __init__(self): super().__init__() self.title = 'Algal Data Analyser' # Default dimensions self.left = 10 * config.wr self.top = 60 * config.wr self.width = 960 * config.wr self.height = 600 * config.wr logger.debug( 'Creating main window [left:%.2f, top:%.2f, width:%.2f, height:%.2f]' % (self.left, self.top, self.width, self.height)) self.setStyleSheet(styles.main_background) self._createMenuBar() self.initUI()
def apply_changes(self): logger.debug('Trying to set colour %s and style %s' % (self.line_colour.text(), self.line_style.currentText())) try: if (is_color_like(self.line_colour.text())): self.data.style["color"] = self.line_colour.text() self.data.style["linestyle"] = self.line_style.currentText() self.data.style["marker"] = config.marker_style_options[ self.marker_style.currentText()] self.parent.set_plot_styles() self.parent.draw() self.close() except Exception as e: logger.exception(e) logger.warning('Unable change style, skipping') pass
def get_all_fit_params(self, signal_name, fit_name, fit_from, fit_to, fit_param): logger.debug('Fitting %s with %s from %.2f to %.2f and recording %s' % ( signal_name, fit_name, fit_from, fit_to, fit_param)) values = [] errors = [] for i, _ in enumerate(self.growth_data.data_files): fit_result, covm = self.get_fit( i, signal_name, fit_name, fit_from, fit_to) param_errors = np.sqrt(np.diag(covm)) model = get_model(fit_name) for i, param in enumerate(model.params): if param == fit_param: values.append(fit_result[i]) errors.append(param_errors[i]) return values, errors
def saveFileDialog(self): logger.debug('Creating window for creating file name for saving') options = QFileDialog.Options() options |= QFileDialog.DontUseNativeDialog self.file_name, _ = QFileDialog.getSaveFileName( self, "Save File", "", "All Files (*);;Text Files (*.txt)", options=options) if self.file_name and self.save_fig: if (self.file_name == ''): self.fig.savefig('graph.png', dpi=1200) elif (self.file_name.find('.') == -1): self.fig.savefig(self.file_name + '.png') else: self.fig.savefig(self.file_name)
def update_condition_data_list(self): logger.debug('Updating the list of condition data files') self.condition_data_list.clear() self.condition_yaxis_dropdown.clear() self.condition_legend_names.clear() for i, data in enumerate(data_manager.get_condition_data_files()): data_list_item = ConditionListItem(data.label, i, self) self.condition_data_list.addItem(data_list_item.item) self.condition_data_list.setItemWidget(data_list_item.item, data_list_item.widget) if data.legend: self.condition_legend_names.addItem(data.legend) else: self.condition_legend_names.addItem(data.label) if i > 0: continue for sig in data.signals: self.condition_yaxis_dropdown.addItem(sig.name)
def __init__(self, purpose='open', fig=None): super().__init__() if purpose == 'open': self.title = 'Open Files' elif purpose == 'save': self.title = 'Save File' elif purpose == 'directory': self.title = 'Choose Directory' self.purpose = purpose if fig: self.fig = fig self.save_fig = True else: self.save_fig = False self.width = 960 * config.wr self.height = 600 * config.hr logger.debug('Creating file handler window [width:%.2f, height:%.2f]' % (self.width, self.height)) self.initUI()
def load(self): logger.debug('Loading files into ADA') file_type = self.file_type.currentText() for file_name in self.files: if file_type == 'Algem Pro' and file_name.endswith('.txt'): self.load_algem_pro(file_name) elif file_type == 'Algem HT24' and file_name.endswith('.txt'): self.load_algem_ht24_txt(file_name) elif file_type == 'Algem HT24' and file_name.endswith('.csv'): self.load_algem_ht24(file_name) elif file_type == 'IP' and file_name.endswith('.csv'): self.load_ip(file_name) elif file_type == 'PSI' and file_name.endswith('.ods'): self.load_psi(file_name) elif file_type == 'ADA' and file_name.endswith('.csv'): self.load_ada(file_name) elif file_type == 'MicrobeMeter' and file_name.endswith('.tsv'): self.load_microbemeter(file_name) else: raise RuntimeError("File %s has the wrong extension" % (file_name)) if len(self.condition_files) > 0: # Set downsampling if option selected downsample = -1 if isint(self.downsample.text()): downsample = int(self.downsample.text()) # Load in optional conditions data for algem machines for file_name in self.condition_files: if file_type == 'Algem Pro' and file_name.endswith('.txt'): self.load_algem_pro_conditions(file_name, downsample) elif file_type == 'Algem HT24' and file_name.endswith('.csv'): self.load_algem_ht24_conditions(file_name, downsample) else: raise RuntimeError("File %s has the wrong extension" % (file_name)) # Update the data lists in the main window self.parent.update_data_list() self.parent.update_condition_data_list() self.close()
def average_data(xdatas, ydatas, show_err=False): logger.debug('Averaging data') new_xdata = np.array([]) new_ydata = np.array([]) new_yerr = np.array([]) if len(xdatas) <= 1: return xdatas[0], ydatas[0], new_yerr for i, x_i in enumerate(xdatas[0]): ys = np.array([ydatas[0][i]]) for j in range(1, len(xdatas), 1): # Interpolate between points to do average ys = np.append(ys, np.interp(x_i, xdatas[j], ydatas[j])) mean = np.mean(ys) std_dev = np.std(ys, ddof=1) if (show_err): std_dev = std_dev / np.sqrt(ys.size) new_xdata = np.append(new_xdata, x_i) new_ydata = np.append(new_ydata, mean) new_yerr = np.append(new_yerr, std_dev) return new_xdata, new_ydata, new_yerr