def select_files_to_plot(self):
current_file = f'{self.working_folder}/{self.list_data.currentItem().text()}'
f = h5py.File(current_file, 'r')
uid_herfds = list(f.keys())
f.close()
hdr = self.db[uid_herfds[0]]
path = hdr.start['interp_filename']
df, header = load_binned_df_from_file(path)
keys = df.keys()
refined_keys = []
for key in keys:
if not (('timestamp' in key) or ('energy' in key)):
refined_keys.append(key)
self.keys = refined_keys
if self.keys != self.last_keys:
self.last_keys = self.keys
self.comboBox_data_numerator.clear()
self.comboBox_data_bkg.clear()
self.comboBox_data_denominator.clear()
self.comboBox_data_numerator.insertItems(0, self.keys)
self.comboBox_data_bkg.insertItems(0, self.keys)
self.comboBox_data_denominator.insertItems(0, self.keys)
if self.last_numerator != '' and self.last_numerator in self.keys:
indx = self.comboBox_data_numerator.findText(
self.last_numerator)
self.comboBox_data_numerator.setCurrentIndex(indx)
if self.last_denominator != '' and self.last_denominator in self.keys:
indx = self.comboBox_data_denominator.findText(
self.last_denominator)
self.comboBox_data_denominator.setCurrentIndex(indx)
if self.last_bkg != '' and self.last_bkg in self.keys:
indx = self.comboBox_data_bkg.findText(self.last_bkg)
self.comboBox_data_bkg.setCurrentIndex(indx)
def add_data_to_project(self):
if self.comboBox_data_numerator.currentText() != -1 and self.comboBox_data_denominator.currentText() != -1:
for item in self.list_data.selectedItems():
filepath = str(Path(self.working_folder) / Path(item.text()))
name = Path(filepath).resolve().stem
df, header = load_binned_df_from_file(filepath)
uid = header[header.find('UID:')+5:header.find('\n', header.find('UID:'))]
try:
md = self.db[uid]['start']
except:
print('Metadata not found')
md={}
df = df.sort_values('energy')
num_key = self.comboBox_data_numerator.currentText()
den_key = self.comboBox_data_denominator.currentText()
mu = df[num_key] / df[den_key]
if self.checkBox_log_bin.checkState():
mu = np.log(mu)
if self.checkBox_inv_bin.checkState():
mu = -mu
mu=np.array(mu)
ds = XASDataSet(name=name,md=md,energy=df['energy'],mu=mu, filename=filepath,datatype='experiment')
ds.header = header
self.parent.project.append(ds)
self.parent.statusBar().showMessage('Scans added to the project successfully')
else:
message_box('Error', 'Select numerator and denominator columns')
def __init__(self, db, fname):
df, header = load_binned_df_from_file(fname)
uid = [i for i in header.split('\n# ') if 'uid' in i][0].split(' ')[-1]
start = ttime.time()
t = db[uid].table(fill=True)
print(f'db read in {ttime.time() - start}')
self.energy = df.energy.values
self.i0 = df.i0.values
self.iff = df.iff.values
self.images = np.array([i.squeeze() for i in t.pil100k_image])
if not np.isclose(t.hhm_energy.values[0], self.energy[0], 1e-4):
self.images = self.images[::-1, :, :]
self.images = self.images/np.abs(self.i0)[:, None, None]
self.muf = self.iff/self.i0
self.total_image = np.sum(self.images, axis=0)
self.y = 0
self.x = 0
self.dy, self.dx = self.total_image.shape
self.energy_converter = None
def plot_xas_data(self):
selected_items = (self.list_data.selectedItems())
update_figure([self.figure_data.ax], self.toolbar, self.canvas)
if self.comboBox_data_numerator.currentText(
) == -1 or self.comboBox_data_denominator.currentText() == -1:
message_box('Warning', 'Please select numerator and denominator')
return
self.last_numerator = self.comboBox_data_numerator.currentText()
self.last_denominator = self.comboBox_data_denominator.currentText()
energy_key = 'energy'
handles = []
for i in selected_items:
path = f'{self.working_folder}/{i.text()}'
print(path)
df, header = load_binned_df_from_file(path)
numer = np.array(df[self.comboBox_data_numerator.currentText()])
denom = np.array(df[self.comboBox_data_denominator.currentText()])
if self.checkBox_ratio.checkState():
y_label = (f'{self.comboBox_data_numerator.currentText()} / '
f'{self.comboBox_data_denominator.currentText()}')
spectrum = numer / denom
else:
y_label = (f'{self.comboBox_data_numerator.currentText()}')
spectrum = numer
if self.checkBox_log_bin.checkState():
spectrum = np.log(spectrum)
y_label = f'ln ({y_label})'
if self.checkBox_inv_bin.checkState():
spectrum = -spectrum
y_label = f'- {y_label}'
self.figure_data.ax.plot(df[energy_key], spectrum)
self.parent.set_figure(self.figure_data.ax,
self.canvas,
label_x='Energy (eV)',
label_y=y_label)
self.figure_data.ax.set_xlabel('Energy (eV)')
self.figure_data.ax.set_ylabel(y_label)
last_trace = self.figure_data.ax.get_lines()[
len(self.figure_data.ax.get_lines()) - 1]
patch = mpatches.Patch(color=last_trace.get_color(),
label=i.text())
handles.append(patch)
self.figure_data.ax.legend(handles=handles)
self.figure_data.tight_layout()
self.canvas.draw_idle()
def get_herfd_data(db, uid):
filename = db[uid].start['interp_filename']
path, extension = os.path.splitext(filename)
if extension != '.dat':
filename = path + '.dat'
print(f' @@@@@@@ Reading {filename}')
df, _ = load_binned_df_from_file(filename)
energy = df['energy'].values
# herfd = np.abs((df[ROI])/ df['i0']).values
# if ROI_bkg is not None:
# herfd_bkg = np.abs((df[ROI_bkg]) / df['i0']).values
# return energy, (herfd-herfd_bkg)
return energy, df
def load_to_dropbox(self, path):
df, header = load_binned_df_from_file(path)
h = header.replace('#', '')
h = h.replace('\n', ',')
d = dict()
for element in h.split(', '):
if ':' in element:
x = element.split(':')
d[x[0]] = x[1]
year, cycle = d['Facility.cycle'].split('-')
proposal = d['Facility.GUP']
dn = '/{}/{}/{}/'.format(year, cycle, proposal).replace(' ', '')
dropbox_upload_files(self.dropbox_service, path, dn,
os.path.basename(path))
def average_scans(path, base, ext=''):
filenames = filenames_from_dir(path, base=base, ext=ext)
header_av = '# merge\n'
path_av = os.path.join(path, base + ' merged' + ext)
# define energy grid and read the data to merge
n = len(filenames)
energy_lo = np.zeros(n)
energy_hi = np.zeros(n)
df_list = []
for i, f in enumerate(filenames):
df, header = load_binned_df_from_file(f)
df_list.append(df)
energy_lo[i] = df.iloc[:, 0].min()
energy_hi[i] = df.iloc[:, 0].max()
_, new_line = os.path.split(f)
header_av += '# ' + new_line + '\n'
energy_lo = energy_lo.max()
energy_hi = energy_hi.min()
E = np.array(df_list[0].iloc[:, 0])
E = E[(E >= energy_lo) & (E <= energy_hi)]
# create new df
idx = pd.Index(np.arange(E.size))
columns = df_list[0].columns
df_av = pd.DataFrame(index=idx, columns=columns)
df_av.iloc[:, 0] = E
df_av.iloc[:, 1:] = 0
# average
for each_df in df_list:
data = np.array(each_df.iloc[:, 1:])
n_cols = data[0, :].size
E_cur = np.array(each_df.iloc[:, 0])
data_int = np.array(
[np.interp(E, E_cur, data[:, i]) for i in range(n_cols)]).T
df_av.iloc[:, 1:] += data_int
df_av.iloc[:, 1:] /= n
columns_str = ' '.join(columns)
fmt = '%12.6f ' + (' '.join(['%12.6e' for i in range(len(columns) - 1)]))
np.savetxt(path_av,
df_av.values,
delimiter=" ",
fmt=fmt,
header=f'# {columns_str}',
comments=header_av)
def select_files_to_plot(self):
df, header = load_binned_df_from_file(f'{self.working_folder}/{self.list_data.currentItem().text()}')
keys = df.keys()
refined_keys = []
for key in keys:
if not (('timestamp' in key) or ('energy' in key)):
refined_keys.append(key)
self.keys = refined_keys
if self.keys != self.last_keys:
self.last_keys = self.keys
self.comboBox_data_numerator.clear()
self.comboBox_data_denominator.clear()
self.comboBox_data_numerator.insertItems(0, self.keys)
self.comboBox_data_denominator.insertItems(0, self.keys)
if self.last_numerator!= '' and self.last_numerator in self.keys:
indx = self.comboBox_data_numerator.findText(self.last_numerator)
self.comboBox_data_numerator.setCurrentIndex(indx)
if self.last_denominator!= '' and self.last_denominator in self.keys:
indx = self.comboBox_data_denominator.findText(self.last_denominator)
self.comboBox_data_denominator.setCurrentIndex(indx)
def get_data():
files = [x.working_folder + '/' + i.text() for i in x.list_data.selectedItems()]
dfs_odd = []
dfs_even = []
for f in files:
_df, _ = load_binned_df_from_file(f)
if 'even' in f:
dfs_even.append(_df)
else:
dfs_odd.append(_df)
data_even = np.vstack([-(_df['pil100_ROI1']/_df['i0']).values for _df in dfs_even]).T
data_odd = np.vstack([-(_df['pil100_ROI1']/_df['i0']).values for _df in dfs_odd]).T
energy_even = dfs_even[0]['energy'].values
energy_odd = dfs_odd[0]['energy'].values
energy = np.hstack((energy_even, energy_odd))
data_even_av = np.mean(data_even, axis=1)
data_odd_av = np.mean(data_odd, axis=1)
data_av = np.hstack((data_even_av, data_odd_av))
plt.figure(); plt.plot(energy_even, '.-'); plt.plot(energy_odd, '.-')
idx_ord = np.argsort(energy)
return energy[idx_ord], data_av[idx_ord]
def generate_output_figures(filepath, imagepath=None, t_flag=True, f_flag=True, r_flag=True):
plt.ioff()
df, header = load_binned_df_from_file(filepath)
df = df.sort_values('energy')
energy = np.array(df['energy'])
mu_t = np.array(np.log(df['i0']/df['it']))
mu_f = np.array(df['iff']/df['i0'])
mu_r = np.array(np.log(df['it']/df['ir']))
try:
ds_t = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_t, filename=filepath, datatype='experiment')
except:
ds_t = None
try:
ds_f = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_f, filename=filepath, datatype='experiment')
except:
ds_f = None
try:
ds_r = XASDataSet(name=filepath, md={}, energy=energy, mu=mu_r, filename=filepath, datatype='experiment')
except:
ds_r = None
fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots(3, 3, figsize=(12, 9))
fig.set_tight_layout(True)
ax_e = (ax1, ax2, ax3, ax4, ax5, ax6)
ax_k = (ax7, ax8, ax9)
ax_mu_raw = (ax1, ax2, ax3)
ax_mu_flat = (ax4, ax5, ax6)
ax_chi = (ax7, ax8, ax9)
if t_flag:
plot_xas_raw(energy, mu_t, ax1, color='b')
plot_xas_in_E(ds_t, ax4, color='b')
plot_xas_in_K(ds_t, ax7, color='b')
if f_flag:
plot_xas_raw(energy, mu_f, ax2, color='r')
plot_xas_in_E(ds_f, ax5, color='r')
plot_xas_in_K(ds_f, ax8, color='r')
if r_flag:
plot_xas_raw(energy, mu_r, ax3, color='k')
plot_xas_in_E(ds_r, ax6, color='k')
plot_xas_in_K(ds_r, ax9, color='k')
for ax in ax_e:
ax.set_xlabel('E, eV')
ax.set_xlim(energy[0], energy[-1])
for ax in ax_k:
ax.set_xlabel('k, A$^{-1}$')
if ds_t:
ax.set_xlim(ds_t.k[0], ds_t.k[-1])
for ax in ax_mu_raw:
ax.set_ylabel('mu')
for ax in ax_mu_flat:
ax.set_ylabel('mu norm')
for ax in ax_chi:
ax.set_ylabel('$\chi$(k) * k$^{2}$')
ax1.set_title('Transmission')
ax2.set_title('Fluorescence')
ax3.set_title('Reference')
# legend = []
#
#
# ax1.legend(legend)
#
# ax1.set_xlabel('E, eV')
# ax1.set_ylabel('norm/flat mu')
# ax3.set_xlabel('E, eV')
# ax3.set_ylabel('norm/flat mu')
# ax5.set_xlabel('E, eV')
# ax5.set_ylabel('norm/flat mu')
#
# ax2.set_xlabel('k, A$^{-1}$')
# ax2.set_ylabel('$\chi$(k) * k$^{2}$')
# ax4.set_xlabel('k, A$^{-1}$')
# ax4.set_ylabel('$\chi$(k) * k$^{2}$')
# ax5.set_xlabel('k, A$^{-1}$')
# ax6.set_ylabel('$\chi$(k) * k$^{2}$')
# plt.tight_layout()
if imagepath:
plt.savefig(imagepath, dpi=300)
plt.ion()
plt.close(fig)
import h5py
from pathlib import Path
from xas.file_io import load_binned_df_from_file
results = list(db.v2.search({'element': 'Co'}))
hfile = h5py.File('/nsls2/xf08id/Sandbox/database/Co_ISS_data.h5', 'w')
for i, result in enumerate(results):
try:
print(i, result)
path = db[result].start['interp_filename']
fn = Path(path)
filename = os.path.basename(path)
nfn = fn.with_suffix('.dat')
df, metadata = load_binned_df_from_file(nfn)
energy = df['energy']
ref = np.log(df['it'] / df['ir'])
trans = np.log(df['i0'] / df['it'])
fluo = (df['iff'] / df['i0'])
hfile.create_group(str(i))
hfile.create_dataset(f'{str(i)}/energy', data=energy)
hfile.create_dataset(f'{str(i)}/ref', data=ref)
hfile.create_dataset(f'{str(i)}/trans', data=trans)
hfile.create_dataset(f'{str(i)}/fluo', data=fluo)
hfile.create_dataset(f'{str(i)}/filename', data=filename)
except:
print(f'{result} failed to load')
# fkey = f.split(' ')[3]
#
# df, header = load_binned_df_from_file(fpath)
# uid = header.split('\n# ')[-3].split(' ')[1]
# hdr = db[uid]
# t = hdr.table(fill=True)
# for f in [files[0]]:
for f in files:
# df, header = load_interpolated_df_from_file(f)
print(f'Processing file: {f}')
fpath = (working_folder + '/' + f)
# fkey = f.split(' ')[3]
fkey = keys[[k in f for k in keys].index(True)]
df, header = load_binned_df_from_file(fpath)
uid = header.split('\n# ')[-3].split(' ')[1]
hdr = db[uid]
t = hdr.table(fill=True)
roi1 = []
roi1_no_bkg = []
for i in t.index:
_roi_sum = pil100k_roi_sum(t.pil100k_image[i].squeeze(), roi_vals)
_roi_sum_no_bkg = pil100k_roi_sum_w_bkg_removed(t.pil100k_image[i].squeeze(), roi_vals, dw=20)
roi1.append(_roi_sum)
roi1_no_bkg.append(_roi_sum_no_bkg)
roi1 = np.array(roi1)
roi1_no_bkg = np.array(roi1_no_bkg)
