def select_files_to_plot(self): current_file = f'{self.working_folder}/{self.list_data.currentItem().text()}' f = h5py.File(current_file, 'r') uid_herfds = list(f.keys()) f.close() hdr = self.db[uid_herfds[0]] path = hdr.start['interp_filename'] df, header = load_binned_df_from_file(path) keys = df.keys() refined_keys = [] for key in keys: if not (('timestamp' in key) or ('energy' in key)): refined_keys.append(key) self.keys = refined_keys if self.keys != self.last_keys: self.last_keys = self.keys self.comboBox_data_numerator.clear() self.comboBox_data_bkg.clear() self.comboBox_data_denominator.clear() self.comboBox_data_numerator.insertItems(0, self.keys) self.comboBox_data_bkg.insertItems(0, self.keys) self.comboBox_data_denominator.insertItems(0, self.keys) if self.last_numerator != '' and self.last_numerator in self.keys: indx = self.comboBox_data_numerator.findText( self.last_numerator) self.comboBox_data_numerator.setCurrentIndex(indx) if self.last_denominator != '' and self.last_denominator in self.keys: indx = self.comboBox_data_denominator.findText( self.last_denominator) self.comboBox_data_denominator.setCurrentIndex(indx) if self.last_bkg != '' and self.last_bkg in self.keys: indx = self.comboBox_data_bkg.findText(self.last_bkg) self.comboBox_data_bkg.setCurrentIndex(indx)
def add_data_to_project(self): if self.comboBox_data_numerator.currentText() != -1 and self.comboBox_data_denominator.currentText() != -1: for item in self.list_data.selectedItems(): filepath = str(Path(self.working_folder) / Path(item.text())) name = Path(filepath).resolve().stem df, header = load_binned_df_from_file(filepath) uid = header[header.find('UID:')+5:header.find('\n', header.find('UID:'))] try: md = self.db[uid]['start'] except: print('Metadata not found') md={} df = df.sort_values('energy') num_key = self.comboBox_data_numerator.currentText() den_key = self.comboBox_data_denominator.currentText() mu = df[num_key] / df[den_key] if self.checkBox_log_bin.checkState(): mu = np.log(mu) if self.checkBox_inv_bin.checkState(): mu = -mu mu=np.array(mu) ds = XASDataSet(name=name,md=md,energy=df['energy'],mu=mu, filename=filepath,datatype='experiment') ds.header = header self.parent.project.append(ds) self.parent.statusBar().showMessage('Scans added to the project successfully') else: message_box('Error', 'Select numerator and denominator columns')
def __init__(self, db, fname): df, header = load_binned_df_from_file(fname) uid = [i for i in header.split('\n# ') if 'uid' in i][0].split(' ')[-1] start = ttime.time() t = db[uid].table(fill=True) print(f'db read in {ttime.time() - start}') self.energy = df.energy.values self.i0 = df.i0.values self.iff = df.iff.values self.images = np.array([i.squeeze() for i in t.pil100k_image]) if not np.isclose(t.hhm_energy.values[0], self.energy[0], 1e-4): self.images = self.images[::-1, :, :] self.images = self.images/np.abs(self.i0)[:, None, None] self.muf = self.iff/self.i0 self.total_image = np.sum(self.images, axis=0) self.y = 0 self.x = 0 self.dy, self.dx = self.total_image.shape self.energy_converter = None
def plot_xas_data(self): selected_items = (self.list_data.selectedItems()) update_figure([self.figure_data.ax], self.toolbar, self.canvas) if self.comboBox_data_numerator.currentText( ) == -1 or self.comboBox_data_denominator.currentText() == -1: message_box('Warning', 'Please select numerator and denominator') return self.last_numerator = self.comboBox_data_numerator.currentText() self.last_denominator = self.comboBox_data_denominator.currentText() energy_key = 'energy' handles = [] for i in selected_items: path = f'{self.working_folder}/{i.text()}' print(path) df, header = load_binned_df_from_file(path) numer = np.array(df[self.comboBox_data_numerator.currentText()]) denom = np.array(df[self.comboBox_data_denominator.currentText()]) if self.checkBox_ratio.checkState(): y_label = (f'{self.comboBox_data_numerator.currentText()} / ' f'{self.comboBox_data_denominator.currentText()}') spectrum = numer / denom else: y_label = (f'{self.comboBox_data_numerator.currentText()}') spectrum = numer if self.checkBox_log_bin.checkState(): spectrum = np.log(spectrum) y_label = f'ln ({y_label})' if self.checkBox_inv_bin.checkState(): spectrum = -spectrum y_label = f'- {y_label}' self.figure_data.ax.plot(df[energy_key], spectrum) self.parent.set_figure(self.figure_data.ax, self.canvas, label_x='Energy (eV)', label_y=y_label) self.figure_data.ax.set_xlabel('Energy (eV)') self.figure_data.ax.set_ylabel(y_label) last_trace = self.figure_data.ax.get_lines()[ len(self.figure_data.ax.get_lines()) - 1] patch = mpatches.Patch(color=last_trace.get_color(), label=i.text()) handles.append(patch) self.figure_data.ax.legend(handles=handles) self.figure_data.tight_layout() self.canvas.draw_idle()
def get_herfd_data(db, uid): filename = db[uid].start['interp_filename'] path, extension = os.path.splitext(filename) if extension != '.dat': filename = path + '.dat' print(f' @@@@@@@ Reading {filename}') df, _ = load_binned_df_from_file(filename) energy = df['energy'].values # herfd = np.abs((df[ROI])/ df['i0']).values # if ROI_bkg is not None: # herfd_bkg = np.abs((df[ROI_bkg]) / df['i0']).values # return energy, (herfd-herfd_bkg) return energy, df
def load_to_dropbox(self, path): df, header = load_binned_df_from_file(path) h = header.replace('#', '') h = h.replace('\n', ',') d = dict() for element in h.split(', '): if ':' in element: x = element.split(':') d[x[0]] = x[1] year, cycle = d['Facility.cycle'].split('-') proposal = d['Facility.GUP'] dn = '/{}/{}/{}/'.format(year, cycle, proposal).replace(' ', '') dropbox_upload_files(self.dropbox_service, path, dn, os.path.basename(path))
def average_scans(path, base, ext=''): filenames = filenames_from_dir(path, base=base, ext=ext) header_av = '# merge\n' path_av = os.path.join(path, base + ' merged' + ext) # define energy grid and read the data to merge n = len(filenames) energy_lo = np.zeros(n) energy_hi = np.zeros(n) df_list = [] for i, f in enumerate(filenames): df, header = load_binned_df_from_file(f) df_list.append(df) energy_lo[i] = df.iloc[:, 0].min() energy_hi[i] = df.iloc[:, 0].max() _, new_line = os.path.split(f) header_av += '# ' + new_line + '\n' energy_lo = energy_lo.max() energy_hi = energy_hi.min() E = np.array(df_list[0].iloc[:, 0]) E = E[(E >= energy_lo) & (E <= energy_hi)] # create new df idx = pd.Index(np.arange(E.size)) columns = df_list[0].columns df_av = pd.DataFrame(index=idx, columns=columns) df_av.iloc[:, 0] = E df_av.iloc[:, 1:] = 0 # average for each_df in df_list: data = np.array(each_df.iloc[:, 1:]) n_cols = data[0, :].size E_cur = np.array(each_df.iloc[:, 0]) data_int = np.array( [np.interp(E, E_cur, data[:, i]) for i in range(n_cols)]).T df_av.iloc[:, 1:] += data_int df_av.iloc[:, 1:] /= n columns_str = ' '.join(columns) fmt = '%12.6f ' + (' '.join(['%12.6e' for i in range(len(columns) - 1)])) np.savetxt(path_av, df_av.values, delimiter=" ", fmt=fmt, header=f'# {columns_str}', comments=header_av)
def select_files_to_plot(self): df, header = load_binned_df_from_file(f'{self.working_folder}/{self.list_data.currentItem().text()}') keys = df.keys() refined_keys = [] for key in keys: if not (('timestamp' in key) or ('energy' in key)): refined_keys.append(key) self.keys = refined_keys if self.keys != self.last_keys: self.last_keys = self.keys self.comboBox_data_numerator.clear() self.comboBox_data_denominator.clear() self.comboBox_data_numerator.insertItems(0, self.keys) self.comboBox_data_denominator.insertItems(0, self.keys) if self.last_numerator!= '' and self.last_numerator in self.keys: indx = self.comboBox_data_numerator.findText(self.last_numerator) self.comboBox_data_numerator.setCurrentIndex(indx) if self.last_denominator!= '' and self.last_denominator in self.keys: indx = self.comboBox_data_denominator.findText(self.last_denominator) self.comboBox_data_denominator.setCurrentIndex(indx)
def get_data(): files = [x.working_folder + '/' + i.text() for i in x.list_data.selectedItems()] dfs_odd = [] dfs_even = [] for f in files: _df, _ = load_binned_df_from_file(f) if 'even' in f: dfs_even.append(_df) else: dfs_odd.append(_df) data_even = np.vstack([-(_df['pil100_ROI1']/_df['i0']).values for _df in dfs_even]).T data_odd = np.vstack([-(_df['pil100_ROI1']/_df['i0']).values for _df in dfs_odd]).T energy_even = dfs_even[0]['energy'].values energy_odd = dfs_odd[0]['energy'].values energy = np.hstack((energy_even, energy_odd)) data_even_av = np.mean(data_even, axis=1) data_odd_av = np.mean(data_odd, axis=1) data_av = np.hstack((data_even_av, data_odd_av)) plt.figure(); plt.plot(energy_even, '.-'); plt.plot(energy_odd, '.-') idx_ord = np.argsort(energy) return energy[idx_ord], data_av[idx_ord]
def generate_output_figures(filepath, imagepath=None, t_flag=True, f_flag=True, r_flag=True): plt.ioff() df, header = load_binned_df_from_file(filepath) df = df.sort_values('energy') energy = np.array(df['energy']) mu_t = np.array(np.log(df['i0']/df['it'])) mu_f = np.array(df['iff']/df['i0']) mu_r = np.array(np.log(df['it']/df['ir'])) try: ds_t = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_t, filename=filepath, datatype='experiment') except: ds_t = None try: ds_f = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_f, filename=filepath, datatype='experiment') except: ds_f = None try: ds_r = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_r, filename=filepath, datatype='experiment') except: ds_r = None fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots(3, 3, figsize=(12, 9)) fig.set_tight_layout(True) ax_e = (ax1, ax2, ax3, ax4, ax5, ax6) ax_k = (ax7, ax8, ax9) ax_mu_raw = (ax1, ax2, ax3) ax_mu_flat = (ax4, ax5, ax6) ax_chi = (ax7, ax8, ax9) if t_flag: plot_xas_raw(energy, mu_t, ax1, color='b') plot_xas_in_E(ds_t, ax4, color='b') plot_xas_in_K(ds_t, ax7, color='b') if f_flag: plot_xas_raw(energy, mu_f, ax2, color='r') plot_xas_in_E(ds_f, ax5, color='r') plot_xas_in_K(ds_f, ax8, color='r') if r_flag: plot_xas_raw(energy, mu_r, ax3, color='k') plot_xas_in_E(ds_r, ax6, color='k') plot_xas_in_K(ds_r, ax9, color='k') for ax in ax_e: ax.set_xlabel('E, eV') ax.set_xlim(energy[0], energy[-1]) for ax in ax_k: ax.set_xlabel('k, A$^{-1}$') if ds_t: ax.set_xlim(ds_t.k[0], ds_t.k[-1]) for ax in ax_mu_raw: ax.set_ylabel('mu') for ax in ax_mu_flat: ax.set_ylabel('mu norm') for ax in ax_chi: ax.set_ylabel('$\chi$(k) * k$^{2}$') ax1.set_title('Transmission') ax2.set_title('Fluorescence') ax3.set_title('Reference') # legend = [] # # # ax1.legend(legend) # # ax1.set_xlabel('E, eV') # ax1.set_ylabel('norm/flat mu') # ax3.set_xlabel('E, eV') # ax3.set_ylabel('norm/flat mu') # ax5.set_xlabel('E, eV') # ax5.set_ylabel('norm/flat mu') # # ax2.set_xlabel('k, A$^{-1}$') # ax2.set_ylabel('$\chi$(k) * k$^{2}$') # ax4.set_xlabel('k, A$^{-1}$') # ax4.set_ylabel('$\chi$(k) * k$^{2}$') # ax5.set_xlabel('k, A$^{-1}$') # ax6.set_ylabel('$\chi$(k) * k$^{2}$') # plt.tight_layout() if imagepath: plt.savefig(imagepath, dpi=300) plt.ion() plt.close(fig)
import h5py from pathlib import Path from xas.file_io import load_binned_df_from_file results = list(db.v2.search({'element': 'Co'})) hfile = h5py.File('/nsls2/xf08id/Sandbox/database/Co_ISS_data.h5', 'w') for i, result in enumerate(results): try: print(i, result) path = db[result].start['interp_filename'] fn = Path(path) filename = os.path.basename(path) nfn = fn.with_suffix('.dat') df, metadata = load_binned_df_from_file(nfn) energy = df['energy'] ref = np.log(df['it'] / df['ir']) trans = np.log(df['i0'] / df['it']) fluo = (df['iff'] / df['i0']) hfile.create_group(str(i)) hfile.create_dataset(f'{str(i)}/energy', data=energy) hfile.create_dataset(f'{str(i)}/ref', data=ref) hfile.create_dataset(f'{str(i)}/trans', data=trans) hfile.create_dataset(f'{str(i)}/fluo', data=fluo) hfile.create_dataset(f'{str(i)}/filename', data=filename) except: print(f'{result} failed to load')
# fkey = f.split(' ')[3] # # df, header = load_binned_df_from_file(fpath) # uid = header.split('\n# ')[-3].split(' ')[1] # hdr = db[uid] # t = hdr.table(fill=True) # for f in [files[0]]: for f in files: # df, header = load_interpolated_df_from_file(f) print(f'Processing file: {f}') fpath = (working_folder + '/' + f) # fkey = f.split(' ')[3] fkey = keys[[k in f for k in keys].index(True)] df, header = load_binned_df_from_file(fpath) uid = header.split('\n# ')[-3].split(' ')[1] hdr = db[uid] t = hdr.table(fill=True) roi1 = [] roi1_no_bkg = [] for i in t.index: _roi_sum = pil100k_roi_sum(t.pil100k_image[i].squeeze(), roi_vals) _roi_sum_no_bkg = pil100k_roi_sum_w_bkg_removed(t.pil100k_image[i].squeeze(), roi_vals, dw=20) roi1.append(_roi_sum) roi1_no_bkg.append(_roi_sum_no_bkg) roi1 = np.array(roi1) roi1_no_bkg = np.array(roi1_no_bkg)