def get_data_df(dataset_name, runs, selection=""): # create a DataFrame df_orig = pd.DataFrame(columns=list(daq_labels.keys()), ) failed_runs = [] runs = sorted(runs) for run in runs: mydict = {} # temporary dict, where to store data fname = DIR + str(run) + "_roi.h5" # the file name try: f = h5py.File(fname, "r") main_dset = f["run_" + str(run)] except: print("Error loading run %s: %s" % (run, sys.exc_info()[1])) failed_runs.append(run) pass # Loading data from the specified datasets for k, v in daq_labels.items(): if k == "delay": # delays are in motor steps mydict[k] = sacla_converter.convert("delay", main_dset[v][:], t0=t0) elif k == "photon_mono_energy": # mono energy settings are in motor steps mydict[k] = sacla_converter.convert("energy", main_dset[v][:]) elif k == "photon_sase_energy": mydict[k + "_mean"] = main_dset[v][:].mean() else: mydict[k] = main_dset[v][:] tmp_df = pd.DataFrame(data=mydict) tmp_df["run"] = run # Append the data to the dataframe df_orig = df_orig.append(tmp_df) # removing failed runs for r in failed_runs: runs.remove(r) # round mono energy and delay df_orig.photon_mono_energy = np.round(df_orig.photon_mono_energy.values, decimals=4) df_orig.delay = np.round(df_orig.delay.values, decimals=1) # create total I0 and absorption coefficients df_orig["I0"] = df_orig.I0_up + df_orig.I0_down df_orig["absorp"] = df_orig.TFY / df_orig.I0 df_orig["is_laser"] = (df_orig['laser_status'] == 1) # set tag number as index df_orig = df_orig.set_index("tags") # filtering out garbage if selection != "": df = df_orig.query(selection) else: df = df_orig # print selection efficiency print("\nSelection efficiency") sel_eff = pd.DataFrame({ "Total": df_orig.groupby("run").count().ND, "Selected": df.groupby("run").count().ND, "Eff.": df.groupby("run").count().ND / df_orig.groupby("run").count().ND }) print(sel_eff) # checking delay settings g = df.groupby(['run', 'delay', 'photon_mono_energy']) print("\nEvents per run and delay settings") print(g.count().TFY) return df, runs
spectra_on = [] ax = [] fig = plt.figure(figsize=(10, 10)) photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:] is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:] is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:] tags = f["/run_" + run + "/event_info/tag_number_list"][:] is_laser_off = is_laser == 0 is_laser_on = is_laser == 1 iol = np.array(f["/run_" + run + IOlow][:]) iou = np.array(f["/run_" + run + IOup][:]) spd = np.array(f["/run_" + run + PDSample][:]) mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:])) nd = np.array(f["/run_" + run + ND]) delay = np.array(f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:]) delay = btc.convert("delay", delay) is_data = (is_xray == 1) * (photon_energy > 9651) * (photon_energy < 9700) * (iol < 0.5) * (iou < 0.5) * (iol > 0.) * (iou > 0.) * (nd > -1) itot = iol[is_data] + iou[is_data] spd = spd[is_data][itot > 0] mono = mono[is_data][(itot > 0)] delay = delay[is_data][(itot > 0)] is_laser = is_laser[is_data][(itot > 0)] nd = nd[is_data][(itot > 0)] itot = itot[itot > 0] absorp = spd / itot
def compute_xas(scan_type, start_run, end_run, data_dir, t0=0): """ load HDF5 files corresponding to runs, filter data and return datasets (using pandas DataFrames) :param scan_type: can be any quantity which is configured, e.g. energy, delay :param start run: first run of the scan :param end_run: last run of the scan """ df = None df_conds = None index_name = scan_type for i in range(int(start_run), int(end_run) + 1): # fname = dir + str(i) + "_nompccd.h5" fname = data_dir + str(i) + "_roi.h5" run = fname.split("/")[-1].replace("_roi", "").replace(".h5", "").replace( "_nompccd", "") try: f = h5py.File(fname, "r") tags = f["/run_" + run + "/event_info/tag_number_list"][:] except IOError: print(exc_info()) continue except: print(exc_info()) #print "Last good run was %d" % int(i - 1) #end_run = str(i - 1) #continue print("[ERROR] dunno what to do, call support!") #break # create dataframes from hdf5 files photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:] is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:] is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:] iol = np.array(f["/run_" + run + IOlow][:]) iou = np.array(f["/run_" + run + IOup][:]) spd = np.array(f["/run_" + run + PDSample][:]) mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:])) nd = np.array(f["/run_" + run + ND]) delay = np.array( f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:]) delay = btc.convert("delay", delay, t0=t0) # Data filtering - to be changed depending on exp. conditions is_data = (is_xray == 1) * (photon_energy > 9600) * (iol < 0.5) * ( iou < 0.5) * (iol > 0.01) * (iou > 0.01) * (nd > -1) # Applying the filter itot = iol[is_data] + iou[is_data] spd = spd[is_data][itot > 0] mono = mono[is_data][(itot > 0)] delay = delay[is_data][(itot > 0)] is_laser = is_laser[is_data][(itot > 0)] nd = nd[is_data][(itot > 0)] itot = itot[itot > 0] tags = tags[is_data] photon_energy = photon_energy[is_data] iou = iou[is_data] iol = iol[is_data] # Calculating the absorption coeff. absorp = spd / itot # Create a simple dictionary with the interesting data data_df = { "energy": mono, "laser": is_laser, "absorp": absorp, "delay": delay, "ND": nd } # Create dataframes if df is None: df = pd.DataFrame(data_df, ) df = df.set_index(index_name) else: df = pd.concat([df, pd.DataFrame(data_df, ).set_index(index_name)]) # Monitoring experimental conditions - in function of puls number if df_conds is None: df_conds = pd.DataFrame( { "tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, ) df_conds = df_conds.set_index("tags") else: df_conds = pd.concat([ df_conds, pd.DataFrame( { "tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, ).set_index("tags") ]) return df, df_conds, end_run
def get_data_df(dataset_name, runs, selection=""): # create a DataFrame df_orig = pd.DataFrame(columns=daq_labels.keys(), ) failed_runs = [] runs = sorted(runs) for run in runs: mydict = {} # temporary dict, where to store data fname = DIR + str(run) +"_roi.h5" # the file name try: f = h5py.File(fname, "r") main_dset = f["run_" + str(run)] except: print "Error loading run %s: %s" % (run, sys.exc_info()[1]) failed_runs.append(run) pass # Loading data from the specified datasets for k, v in daq_labels.iteritems(): if k == "delay": # delays are in motor steps mydict[k] = sacla_converter.convert("delay", main_dset[v][:], t0=t0) elif k == "photon_mono_energy": # mono energy settings are in motor steps mydict[k] = sacla_converter.convert("energy", main_dset[v][:]) elif k == "photon_sase_energy": mydict[k + "_mean"] = main_dset[v][:].mean() else: mydict[k] = main_dset[v][:] tmp_df = pd.DataFrame(data=mydict) tmp_df["run"] = run # Append the data to the dataframe df_orig = df_orig.append(tmp_df) # removing failed runs for r in failed_runs: runs.remove(r) # round mono energy and delay df_orig.photon_mono_energy = np.round(df_orig.photon_mono_energy.values, decimals=4) df_orig.delay = np.round(df_orig.delay.values, decimals=1) # create total I0 and absorption coefficients df_orig["I0"] = df_orig.I0_up + df_orig.I0_down df_orig["absorp"] = df_orig.TFY / df_orig.I0 df_orig["is_laser"] = (df_orig['laser_status'] == 1) # set tag number as index df_orig = df_orig.set_index("tags") # filtering out garbage if selection != "": df = df_orig.query(selection) else: df = df_orig # print selection efficiency print "\nSelection efficiency" sel_eff = pd.DataFrame( {"Total":df_orig.groupby("run").count().ND, "Selected": df.groupby("run").count().ND, "Eff.": df.groupby("run").count().ND / df_orig.groupby("run").count().ND}) print sel_eff # checking delay settings g = df.groupby(['run', 'delay', 'photon_mono_energy']) print "\nEvents per run and delay settings" print g.count().TFY return df, runs
photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:] is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:] is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:] tags = f["/run_" + run + "/event_info/tag_number_list"][:] is_laser_off = is_laser == 0 is_laser_on = is_laser == 1 iol = np.array(f["/run_" + run + IOlow][:]) iou = np.array(f["/run_" + run + IOup][:]) spd = np.array(f["/run_" + run + PDSample][:]) mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:])) nd = np.array(f["/run_" + run + ND]) delay = np.array(f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:]) delay = btc.convert("delay", delay) is_data = (is_xray == 1) * (photon_energy > 9651) * ( photon_energy < 9700) * (iol < 0.5) * (iou < 0.5) * (iol > 0.) * (iou > 0.) * (nd > -1) itot = iol[is_data] + iou[is_data] spd = spd[is_data][itot > 0] mono = mono[is_data][(itot > 0)] delay = delay[is_data][(itot > 0)] is_laser = is_laser[is_data][(itot > 0)] nd = nd[is_data][(itot > 0)]
def compute_xas(scan_type, start_run, end_run, data_dir, t0=0): """ load HDF5 files corresponding to runs, filter data and return datasets (using pandas DataFrames) :param scan_type: can be any quantity which is configured, e.g. energy, delay :param start run: first run of the scan :param end_run: last run of the scan """ df = None df_conds = None index_name = scan_type for i in range(int(start_run), int(end_run) + 1): # fname = dir + str(i) + "_nompccd.h5" fname = data_dir + str(i) + "_roi.h5" run = fname.split("/")[-1].replace("_roi", "").replace(".h5", "").replace("_nompccd", "") try: f = h5py.File(fname, "r") tags = f["/run_" + run + "/event_info/tag_number_list"][:] except IOError: print exc_info() continue except: print exc_info() #print "Last good run was %d" % int(i - 1) #end_run = str(i - 1) #continue print "[ERROR] dunno what to do, call support!" #break # create dataframes from hdf5 files photon_energy = f["/run_" + run + "/event_info/bl_3/oh_2/photon_energy_in_eV"][:] is_xray = f["/run_" + run + "/event_info/bl_3/eh_1/xfel_pulse_selector_status"][:] is_laser = f["/run_" + run + "/event_info/bl_3/lh_1/laser_pulse_selector_status"][:] iol = np.array(f["/run_" + run + IOlow][:]) iou = np.array(f["/run_" + run + IOup][:]) spd = np.array(f["/run_" + run + PDSample][:]) mono = btc.convert("energy", np.array(f["/run_" + run + Mono][:])) nd = np.array(f["/run_" + run + ND]) delay = np.array(f["/run_" + run + "/event_info/bl_3/eh_4/laser/delay_line_motor_29"][:]) delay = btc.convert("delay", delay, t0=t0) # Data filtering - to be changed depending on exp. conditions is_data = (is_xray == 1) * (photon_energy > 9600) * (iol < 0.5) * (iou < 0.5) * (iol > 0.01) * (iou > 0.01) * (nd > -1) # Applying the filter itot = iol[is_data] + iou[is_data] spd = spd[is_data][itot > 0] mono = mono[is_data][(itot > 0)] delay = delay[is_data][(itot > 0)] is_laser = is_laser[is_data][(itot > 0)] nd = nd[is_data][(itot > 0)] itot = itot[itot > 0] tags = tags[is_data] photon_energy = photon_energy[is_data] iou = iou[is_data] iol = iol[is_data] # Calculating the absorption coeff. absorp = spd / itot # Create a simple dictionary with the interesting data data_df = {"energy": mono, "laser": is_laser, "absorp": absorp, "delay": delay, "ND": nd} # Create dataframes if df is None: df = pd.DataFrame(data_df, ) df = df.set_index(index_name) else: df = pd.concat([df, pd.DataFrame(data_df, ).set_index(index_name)]) # Monitoring experimental conditions - in function of puls number if df_conds is None: df_conds = pd.DataFrame({"tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, ) df_conds = df_conds.set_index("tags") else: df_conds = pd.concat([df_conds, pd.DataFrame({"tags": tags, "photon_energy": photon_energy, "I0up": iou, "I0down": iol, }, ).set_index("tags")]) return df, df_conds, end_run