def get_data_df(dataset_name, runs, selection=""):
# create a DataFrame
df_orig = pd.DataFrame(columns=list(daq_labels.keys()), )
failed_runs = []
runs = sorted(runs)
for run in runs:
mydict = {} # temporary dict, where to store data
fname = DIR + str(run) + "_roi.h5" # the file name
try:
f = h5py.File(fname, "r")
main_dset = f["run_" + str(run)]
except:
print("Error loading run %s: %s" % (run, sys.exc_info()[1]))
failed_runs.append(run)
pass
# Loading data from the specified datasets
for k, v in daq_labels.items():
if k == "delay":
# delays are in motor steps
mydict[k] = sacla_converter.convert("delay",
main_dset[v][:],
t0=t0)
elif k == "photon_mono_energy":
# mono energy settings are in motor steps
mydict[k] = sacla_converter.convert("energy", main_dset[v][:])
elif k == "photon_sase_energy":
mydict[k + "_mean"] = main_dset[v][:].mean()
else:
mydict[k] = main_dset[v][:]
tmp_df = pd.DataFrame(data=mydict)
tmp_df["run"] = run
# Append the data to the dataframe
df_orig = df_orig.append(tmp_df)
# removing failed runs
for r in failed_runs:
runs.remove(r)
# round mono energy and delay
df_orig.photon_mono_energy = np.round(df_orig.photon_mono_energy.values,
decimals=4)
df_orig.delay = np.round(df_orig.delay.values, decimals=1)
# create total I0 and absorption coefficients
df_orig["I0"] = df_orig.I0_up + df_orig.I0_down
df_orig["absorp"] = df_orig.TFY / df_orig.I0
df_orig["is_laser"] = (df_orig['laser_status'] == 1)
# set tag number as index
df_orig = df_orig.set_index("tags")
# filtering out garbage
if selection != "":
df = df_orig.query(selection)
else:
df = df_orig
# print selection efficiency
print("\nSelection efficiency")
sel_eff = pd.DataFrame({
"Total":
df_orig.groupby("run").count().ND,
"Selected":
df.groupby("run").count().ND,
"Eff.":
df.groupby("run").count().ND / df_orig.groupby("run").count().ND
})
print(sel_eff)
# checking delay settings
g = df.groupby(['run', 'delay', 'photon_mono_energy'])
print("\nEvents per run and delay settings")
print(g.count().TFY)
return df, runs
spectra_on = []
ax = []
fig = plt.figure(figsize=(10, 10))
photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:]
is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:]
is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:]
tags = f["/run_" + run + "/event_info/tag_number_list"][:]
is_laser_off = is_laser == 0
is_laser_on = is_laser == 1
iol = np.array(f["/run_" + run + IOlow][:])
iou = np.array(f["/run_" + run + IOup][:])
spd = np.array(f["/run_" + run + PDSample][:])
mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:]))
nd = np.array(f["/run_" + run + ND])
delay = np.array(f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:])
delay = btc.convert("delay", delay)
is_data = (is_xray == 1) * (photon_energy > 9651) * (photon_energy < 9700) * (iol < 0.5) * (iou < 0.5) * (iol > 0.) * (iou > 0.) * (nd > -1)
itot = iol[is_data] + iou[is_data]
spd = spd[is_data][itot > 0]
mono = mono[is_data][(itot > 0)]
delay = delay[is_data][(itot > 0)]
is_laser = is_laser[is_data][(itot > 0)]
nd = nd[is_data][(itot > 0)]
itot = itot[itot > 0]
absorp = spd / itot
def compute_xas(scan_type, start_run, end_run, data_dir, t0=0):
"""
load HDF5 files corresponding to runs, filter data and return datasets (using pandas DataFrames)
:param scan_type: can be any quantity which is configured, e.g. energy, delay
:param start run: first run of the scan
:param end_run: last run of the scan
"""
df = None
df_conds = None
index_name = scan_type
for i in range(int(start_run), int(end_run) + 1):
# fname = dir + str(i) + "_nompccd.h5"
fname = data_dir + str(i) + "_roi.h5"
run = fname.split("/")[-1].replace("_roi",
"").replace(".h5", "").replace(
"_nompccd", "")
try:
f = h5py.File(fname, "r")
tags = f["/run_" + run + "/event_info/tag_number_list"][:]
except IOError:
print(exc_info())
continue
except:
print(exc_info())
#print "Last good run was %d" % int(i - 1)
#end_run = str(i - 1)
#continue
print("[ERROR] dunno what to do, call support!")
#break
# create dataframes from hdf5 files
photon_energy = f["/run_" + run +
"/event_info/bl_3/oh_2/photon_energy_in_eV"][:]
is_xray = f["/run_" + run +
"/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:]
is_laser = f["/run_" + run +
"/event_info/bl_3/lh_1/laser_pulse_selector_status"][:]
iol = np.array(f["/run_" + run + IOlow][:])
iou = np.array(f["/run_" + run + IOup][:])
spd = np.array(f["/run_" + run + PDSample][:])
mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:]))
nd = np.array(f["/run_" + run + ND])
delay = np.array(
f["/run_" + run +
"/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:])
delay = btc.convert("delay", delay, t0=t0)
# Data filtering - to be changed depending on exp. conditions
is_data = (is_xray == 1) * (photon_energy > 9600) * (iol < 0.5) * (
iou < 0.5) * (iol > 0.01) * (iou > 0.01) * (nd > -1)
# Applying the filter
itot = iol[is_data] + iou[is_data]
spd = spd[is_data][itot > 0]
mono = mono[is_data][(itot > 0)]
delay = delay[is_data][(itot > 0)]
is_laser = is_laser[is_data][(itot > 0)]
nd = nd[is_data][(itot > 0)]
itot = itot[itot > 0]
tags = tags[is_data]
photon_energy = photon_energy[is_data]
iou = iou[is_data]
iol = iol[is_data]
# Calculating the absorption coeff.
absorp = spd / itot
# Create a simple dictionary with the interesting data
data_df = {
"energy": mono,
"laser": is_laser,
"absorp": absorp,
"delay": delay,
"ND": nd
}
# Create dataframes
if df is None:
df = pd.DataFrame(data_df, )
df = df.set_index(index_name)
else:
df = pd.concat([df, pd.DataFrame(data_df, ).set_index(index_name)])
# Monitoring experimental conditions - in function of puls number
if df_conds is None:
df_conds = pd.DataFrame(
{
"tags": tags,
"photon_energy": photon_energy,
"I0up": iou,
"I0down": iol,
}, )
df_conds = df_conds.set_index("tags")
else:
df_conds = pd.concat([
df_conds,
pd.DataFrame(
{
"tags": tags,
"photon_energy": photon_energy,
"I0up": iou,
"I0down": iol,
}, ).set_index("tags")
])
return df, df_conds, end_run
def get_data_df(dataset_name, runs, selection=""):
# create a DataFrame
df_orig = pd.DataFrame(columns=daq_labels.keys(), )
failed_runs = []
runs = sorted(runs)
for run in runs:
mydict = {} # temporary dict, where to store data
fname = DIR + str(run) +"_roi.h5" # the file name
try:
f = h5py.File(fname, "r")
main_dset = f["run_" + str(run)]
except:
print "Error loading run %s: %s" % (run, sys.exc_info()[1])
failed_runs.append(run)
pass
# Loading data from the specified datasets
for k, v in daq_labels.iteritems():
if k == "delay":
# delays are in motor steps
mydict[k] = sacla_converter.convert("delay", main_dset[v][:], t0=t0)
elif k == "photon_mono_energy":
# mono energy settings are in motor steps
mydict[k] = sacla_converter.convert("energy", main_dset[v][:])
elif k == "photon_sase_energy":
mydict[k + "_mean"] = main_dset[v][:].mean()
else:
mydict[k] = main_dset[v][:]
tmp_df = pd.DataFrame(data=mydict)
tmp_df["run"] = run
# Append the data to the dataframe
df_orig = df_orig.append(tmp_df)
# removing failed runs
for r in failed_runs:
runs.remove(r)
# round mono energy and delay
df_orig.photon_mono_energy = np.round(df_orig.photon_mono_energy.values, decimals=4)
df_orig.delay = np.round(df_orig.delay.values, decimals=1)
# create total I0 and absorption coefficients
df_orig["I0"] = df_orig.I0_up + df_orig.I0_down
df_orig["absorp"] = df_orig.TFY / df_orig.I0
df_orig["is_laser"] = (df_orig['laser_status'] == 1)
# set tag number as index
df_orig = df_orig.set_index("tags")
# filtering out garbage
if selection != "":
df = df_orig.query(selection)
else:
df = df_orig
# print selection efficiency
print "\nSelection efficiency"
sel_eff = pd.DataFrame( {"Total":df_orig.groupby("run").count().ND,
"Selected": df.groupby("run").count().ND,
"Eff.": df.groupby("run").count().ND / df_orig.groupby("run").count().ND})
print sel_eff
# checking delay settings
g = df.groupby(['run', 'delay', 'photon_mono_energy'])
print "\nEvents per run and delay settings"
print g.count().TFY
return df, runs
photon_energy = f["/run_" + run +
"/event_info/bl_3/oh_2/photon_energy_in_eV"][:]
is_xray = f["/run_" + run +
"/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:]
is_laser = f["/run_" + run +
"/event_info/bl_3/lh_1/laser_pulse_selector_status"][:]
tags = f["/run_" + run + "/event_info/tag_number_list"][:]
is_laser_off = is_laser == 0
is_laser_on = is_laser == 1
iol = np.array(f["/run_" + run + IOlow][:])
iou = np.array(f["/run_" + run + IOup][:])
spd = np.array(f["/run_" + run + PDSample][:])
mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:]))
nd = np.array(f["/run_" + run + ND])
delay = np.array(f["/run_" + run +
"/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:])
delay = btc.convert("delay", delay)
is_data = (is_xray == 1) * (photon_energy > 9651) * (
photon_energy <
9700) * (iol < 0.5) * (iou < 0.5) * (iol > 0.) * (iou > 0.) * (nd > -1)
itot = iol[is_data] + iou[is_data]
spd = spd[is_data][itot > 0]
mono = mono[is_data][(itot > 0)]
delay = delay[is_data][(itot > 0)]
is_laser = is_laser[is_data][(itot > 0)]
nd = nd[is_data][(itot > 0)]
def compute_xas(scan_type, start_run, end_run, data_dir, t0=0):
"""
load HDF5 files corresponding to runs, filter data and return datasets (using pandas DataFrames)
:param scan_type: can be any quantity which is configured, e.g. energy, delay
:param start run: first run of the scan
:param end_run: last run of the scan
"""
df = None
df_conds = None
index_name = scan_type
for i in range(int(start_run), int(end_run) + 1):
# fname = dir + str(i) + "_nompccd.h5"
fname = data_dir + str(i) + "_roi.h5"
run = fname.split("/")[-1].replace("_roi", "").replace(".h5", "").replace("_nompccd", "")
try:
f = h5py.File(fname, "r")
tags = f["/run_" + run + "/event_info/tag_number_list"][:]
except IOError:
print exc_info()
continue
except:
print exc_info()
#print "Last good run was %d" % int(i - 1)
#end_run = str(i - 1)
#continue
print "[ERROR] dunno what to do, call support!"
#break
# create dataframes from hdf5 files
photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:]
is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:]
is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:]
iol = np.array(f["/run_" + run + IOlow][:])
iou = np.array(f["/run_" + run + IOup][:])
spd = np.array(f["/run_" + run + PDSample][:])
mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:]))
nd = np.array(f["/run_" + run + ND])
delay = np.array(f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:])
delay = btc.convert("delay", delay, t0=t0)
# Data filtering - to be changed depending on exp. conditions
is_data = (is_xray == 1) * (photon_energy > 9600) * (iol < 0.5) * (iou < 0.5) * (iol > 0.01) * (iou > 0.01) * (nd > -1)
# Applying the filter
itot = iol[is_data] + iou[is_data]
spd = spd[is_data][itot > 0]
mono = mono[is_data][(itot > 0)]
delay = delay[is_data][(itot > 0)]
is_laser = is_laser[is_data][(itot > 0)]
nd = nd[is_data][(itot > 0)]
itot = itot[itot > 0]
tags = tags[is_data]
photon_energy = photon_energy[is_data]
iou = iou[is_data]
iol = iol[is_data]
# Calculating the absorption coeff.
absorp = spd / itot
# Create a simple dictionary with the interesting data
data_df = {"energy": mono, "laser": is_laser, "absorp": absorp, "delay": delay, "ND": nd}
# Create dataframes
if df is None:
df = pd.DataFrame(data_df, )
df = df.set_index(index_name)
else:
df = pd.concat([df, pd.DataFrame(data_df, ).set_index(index_name)])
# Monitoring experimental conditions - in function of puls number
if df_conds is None:
df_conds = pd.DataFrame({"tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, )
df_conds = df_conds.set_index("tags")
else:
df_conds = pd.concat([df_conds, pd.DataFrame({"tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, ).set_index("tags")])
return df, df_conds, end_run
