def dsp_to_hit(): """ save calibrated energies into the dsp file. this is a good example of adding a column, reading & writing to an LH5 file. """ f_dsp = '/Users/wisecg/Data/OPPI/dsp/oppi_run0_cyc2027_dsp_test.lh5' f_hit = '/Users/wisecg/Data/OPPI/hit/oppi_run0_cyc2027_hit.lh5' sto = lh5.Store() groups = sto.ls(f_dsp) tb_name = 'ORSIS3302DecoderForEnergy/raw' data = sto.read_object(tb_name, f_dsp) df_dsp = data.get_dataframe() # add a new column for each energy estimator of interest for etype in ['energy', 'trapE']: ecal_name = etype + '_cal' pfit = linear_cal(etype) df_dsp[ecal_name] = df_dsp[etype] * pfit[0] + pfit[1] e_cal_lh5 = lh5.Array(df_dsp[ecal_name].values, attrs={'units': 'keV'}) data.add_field(f'{etype}_cal', e_cal_lh5) # write to hit file. delete if exists, LH5 overwrite is broken rn if os.path.exists(f_hit): os.remove(f_hit) sto.write_object(data, tb_name, f_hit)
def raw_to_dsp(ds, overwrite=False, nevt=None, test=False, verbose=2, block=8, group=''): """ Run raw_to_dsp on a set of runs. [raw file] ---> [dsp_run{}.lh5] (digital signal processing results) """ for run in ds.runs: raw_file = "/lfs/l1/legend/users/dandrea/pygama/pgt/tier1/pgt_longtrace_run0117-20200110-105115-calib_raw.lh5" dsp_file = "/lfs/l1/legend/users/dandrea/pygama/pgt/tier2/pgt_longtrace_run0117-20200110-105115-calib_dsp.lh5" #raw_file = ds.paths[run]["raw_path"] #dsp_file = ds.paths[run]["dsp_path"] print("raw_file: ",raw_file) print("dsp_file: ",dsp_file) if dsp_file is not None and overwrite is False: continue if dsp_file is None: # declare new file name dsp_file = raw_file.replace('raw_', 'dsp_') if test: print("test mode (dry run), processing raw file:", raw_file) continue print("Definition of new LH5 version") #f_lh5 = lh5.Store() #data = f_lh5.read_object("raw", raw_file) #wf_in = data['waveform']['values'].nda #dt = data['waveform']['dt'].nda[0] * unit_parser.parse_unit(data['waveform']['dt'].attrs['units']) lh5_in = lh5.Store() #groups = lh5_in.ls(raw_file, group) f = h5py.File(raw_file,'r') print("File info: ",f.keys()) for group in f.keys(): print("Processing: " + raw_file + '/' + group) #data = lh5_in.read_object(group, raw_file) data = f[group]['raw'] #wf_in = data['waveform']['values'].nda #dt = data['waveform']['dt'].nda[0] * unit_parser.parse_unit(data['waveform']['dt'].attrs['units']) wf_in = data['waveform']['values'][()] dt = data['waveform']['dt'][0] * unit_parser.parse_unit(data['waveform']['dt'].attrs['units']) # Parameters for DCR calculation dcr_trap_int = 200 dcr_trap_flat = 1000 dcr_trap_startSample = 1200 # Set up processing chain proc = ProcessingChain(block_width=block, clock_unit=dt, verbosity=verbose) proc.add_input_buffer("wf", wf_in, dtype='float32') # Basic Filters proc.add_processor(mean_stdev, "wf[0:1000]", "bl", "bl_sig") proc.add_processor(np.subtract, "wf", "bl", "wf_blsub") proc.add_processor(pole_zero, "wf_blsub", 145*us, "wf_pz") proc.add_processor(trap_norm, "wf_pz", 10*us, 5*us, "wf_trap") proc.add_processor(asymTrapFilter, "wf_pz", 0.05*us, 2*us, 4*us, "wf_atrap") # Timepoint calculation proc.add_processor(np.argmax, "wf_blsub", 1, "t_max", signature='(n),()->()', types=['fi->i']) proc.add_processor(time_point_frac, "wf_blsub", 0.95, "t_max", "tp_95") proc.add_processor(time_point_frac, "wf_blsub", 0.8, "t_max", "tp_80") proc.add_processor(time_point_frac, "wf_blsub", 0.5, "t_max", "tp_50") proc.add_processor(time_point_frac, "wf_blsub", 0.2, "t_max", "tp_20") proc.add_processor(time_point_frac, "wf_blsub", 0.05, "t_max", "tp_05") proc.add_processor(time_point_thresh, "wf_atrap[0:2000]", 0, "tp_0") # Energy calculation proc.add_processor(np.amax, "wf_trap", 1, "trapEmax", signature='(n),()->()', types=['fi->f']) proc.add_processor(fixed_time_pickoff, "wf_trap", "tp_0+(5*us+9*us)", "trapEftp") proc.add_processor(trap_pickoff, "wf_pz", 1.5*us, 0, "tp_0", "ct_corr") # Current calculation proc.add_processor(avg_current, "wf_pz", 10, "curr(len(wf_pz)-10, f)") proc.add_processor(np.amax, "curr", 1, "curr_amp", signature='(n),()->()', types=['fi->f']) proc.add_processor(np.divide, "curr_amp", "trapEftp", "aoe") # DCR calculation: use slope using 1000 samples apart and averaging 200 # samples, with the start 1.5 us offset from t0 proc.add_processor(trap_pickoff, "wf_pz", 200, 1000, "tp_0+1.5*us", "dcr_unnorm") proc.add_processor(np.divide, "dcr_unnorm", "trapEftp", "dcr") # Tail slope. Basically the same as DCR, except with no PZ correction proc.add_processor(linear_fit, "wf_blsub[3000:]", "wf_b", "wf_m") proc.add_processor(np.divide, "-wf_b", "wf_m", "tail_rc") #add zac filter energy calculation sigma = 10*us flat = 1*us decay = 160*us proc.add_processor(zac_filter, "wf", sigma, flat, decay, "wf_zac(101, f)") proc.add_processor(np.amax, "wf_zac", 1, "zacE", signature='(n),()->()', types=['fi->f']) # Set up the LH5 output lh5_out = lh5.Table(size=proc._buffer_len) lh5_out.add_field("zacE", lh5.Array(proc.get_output_buffer("zacE"), attrs={"units":"ADC"})) lh5_out.add_field("trapEmax", lh5.Array(proc.get_output_buffer("trapEmax"), attrs={"units":"ADC"})) lh5_out.add_field("trapEftp", lh5.Array(proc.get_output_buffer("trapEftp"), attrs={"units":"ADC"})) lh5_out.add_field("ct_corr", lh5.Array(proc.get_output_buffer("ct_corr"), attrs={"units":"ADC*ns"})) lh5_out.add_field("bl", lh5.Array(proc.get_output_buffer("bl"), attrs={"units":"ADC"})) lh5_out.add_field("bl_sig", lh5.Array(proc.get_output_buffer("bl_sig"), attrs={"units":"ADC"})) lh5_out.add_field("A", lh5.Array(proc.get_output_buffer("curr_amp"), attrs={"units":"ADC"})) lh5_out.add_field("AoE", lh5.Array(proc.get_output_buffer("aoe"), attrs={"units":"ADC"})) lh5_out.add_field("dcr", lh5.Array(proc.get_output_buffer("dcr"), attrs={"units":"ADC"})) lh5_out.add_field("tp_max", lh5.Array(proc.get_output_buffer("tp_95", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_95", lh5.Array(proc.get_output_buffer("tp_95", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_80", lh5.Array(proc.get_output_buffer("tp_80", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_50", lh5.Array(proc.get_output_buffer("tp_50", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_20", lh5.Array(proc.get_output_buffer("tp_20", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_05", lh5.Array(proc.get_output_buffer("tp_05", unit=us), attrs={"units":"us"})) lh5_out.add_field("tp_0", lh5.Array(proc.get_output_buffer("tp_0", unit=us), attrs={"units":"us"})) lh5_out.add_field("tail_rc", lh5.Array(proc.get_output_buffer("tail_rc", unit=us), attrs={"units":"us"})) print("Processing:\n",proc) proc.execute() #groupname = group[:group.rfind('/')+1]+"data" groupname = group+"/data" print("Writing to: " + dsp_file + "/" + groupname) lh5_in.write_object(lh5_out, groupname, dsp_file)
def optimize_trap(dg): """ Generate a file with grid points to search, and events from the target peak. Then run DSP a bunch of times on the small table, and fit the peak w/ the peakshape function. NOTE: run table-to-table DSP (no file I/O) """ f_peak = './temp_peak.lh5' # lh5 f_results = './temp_results.h5' # pandas grp_data, grp_grid = '/optimize_data', '/optimize_grid' # epar, elo, ehi, epb = 'energy', 0, 1e7, 10000 # full range epar, elo, ehi, epb = 'energy', 3.88e6, 3.92e6, 500 # K40 peak show_movie = True write_output = True n_rows = None # default None with open('opt_trap.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) # files to consider. fixme: right now only works with one file sto = lh5.Store() lh5_dir = os.path.expandvars(dg.config['lh5_dir']) raw_list = lh5_dir + dg.fileDB['raw_path'] + '/' + dg.fileDB['raw_file'] f_raw = raw_list.values[0] tb_raw = 'ORSIS3302DecoderForEnergy/raw/' # quick check of the energy range # ene_raw = sto.read_object(tb_raw+'/'+epar, f_raw).nda # hist, bins, var = pgh.get_hist(ene_raw, range=(elo, ehi), dx=epb) # plt.plot(bins[1:], hist, ds='steps') # plt.show() # exit() # set grid parameters # TODO: jason's suggestions, knowing the expected shape of the noise curve # e_rises = np.linspace(-1, 0, sqrt(sqrt(3)) # e_rises # make another list which is 10^pwr of this list # np.linspace(log_tau_min, log_tau_max) # try this too e_rises = np.arange(1, 12, 1) e_flats = np.arange(1, 6, 1) # rc_consts = np.arange(54, 154, 10) # changing this here messes up DCR # -- create the grid search file the first time -- # NOTE: this makes a linear grid, and is editable by the arrays above. # jason also proposed a more active gradient-descent style search # like with Brent's method. (https://en.wikipedia.org/wiki/Brent%27s_method) if True: # if not os.path.exists(f_peak): print('Recreating grid search file') # create the grid file # NOTE: save it as an lh5 Table just as an example of writing/reading one lists = [e_rises, e_flats] #, rc_consts] prod = list(itertools.product(*lists)) # clint <3 stackoverflow df_grid = pd.DataFrame(prod, columns=['rise', 'flat']) #,'rc']) lh5_grid = {} for i, dfcol in df_grid.iteritems(): lh5_grid[dfcol.name] = lh5.Array(dfcol.values) tb_grid = lh5.Table(col_dict=lh5_grid) sto.write_object(tb_grid, grp_grid, f_peak) # filter events by onboard energy ene_raw = sto.read_object(tb_raw + '/' + epar, f_raw).nda # hist, bins, var = pgh.get_hist(ene_raw, range=(elo, ehi), dx=epb) # plt.plot(bins[1:], hist, ds='steps') # plt.show() if n_rows is not None: ene_raw = ene_raw[:n_rows] idx = np.where((ene_raw > elo) & (ene_raw < ehi)) # create a filtered table with correct waveform and attrs # TODO: move this into a function in lh5.py which takes idx as an input tb_data, wf_tb_data = lh5.Table(), lh5.Table() # read non-wf cols (lh5 Arrays) data_raw = sto.read_object(tb_raw, f_raw, n_rows=n_rows) for col in data_raw.keys(): if col == 'waveform': continue newcol = lh5.Array(data_raw[col].nda[idx], attrs=data_raw[col].attrs) tb_data.add_field(col, newcol) # handle waveform column (lh5 Table) data_wfs = sto.read_object(tb_raw + '/waveform', f_raw, n_rows=n_rows) for col in data_wfs.keys(): attrs = data_wfs[col].attrs if isinstance(data_wfs[col], lh5.ArrayOfEqualSizedArrays): # idk why i can't put the filtered array into the constructor aoesa = lh5.ArrayOfEqualSizedArrays(attrs=attrs, dims=[1, 1]) aoesa.nda = data_wfs[col].nda[idx] newcol = aoesa else: newcol = lh5.Array(data_wfs[col].nda[idx], attrs=attrs) wf_tb_data.add_field(col, newcol) tb_data.add_field('waveform', wf_tb_data) tb_data.attrs = data_raw.attrs sto.write_object(tb_data, grp_data, f_peak) else: print('Loading peak file. groups:', sto.ls(f_peak)) tb_grid = sto.read_object(grp_grid, f_peak) tb_data = sto.read_object(grp_data, f_peak) # filtered file # tb_data = sto.read_object(tb_raw, f_raw) # orig file df_grid = tb_grid.get_dataframe() # check shape of input table print('input table attributes:') for key in tb_data.keys(): obj = tb_data[key] if isinstance(obj, lh5.Table): for key2 in obj.keys(): obj2 = obj[key2] print(' ', key, key2, obj2.nda.shape, obj2.attrs) else: print(' ', key, obj.nda.shape, obj.attrs) # clear new colums if they exist new_cols = ['e_fit', 'fwhm_fit', 'rchisq', 'xF_err', 'fwhm_ovr_mean'] for col in new_cols: if col in df_grid.columns: df_grid.drop(col, axis=1, inplace=True) t_start = time.time() def run_dsp(dfrow): """ run dsp on the test file, editing the processor list alternate idea: generate a long list of processors with different names """ # adjust dsp config dictionary rise, flat = dfrow # dsp_config['processors']['wf_pz']['defaults']['db.pz.tau'] = f'{tau}*us' dsp_config['processors']['wf_trap']['args'][1] = f'{rise}*us' dsp_config['processors']['wf_trap']['args'][2] = f'{flat}*us' # pprint(dsp_config) # run dsp pc, tb_out = build_processing_chain(tb_data, dsp_config, verbosity=0) pc.execute() # analyze peak e_peak = 1460. etype = 'trapEmax' elo, ehi, epb = 4000, 4500, 3 # the peak moves around a bunch energy = tb_out[etype].nda # get histogram hE, bins, vE = pgh.get_hist(energy, range=(elo, ehi), dx=epb) xE = bins[1:] # should I center the max at 1460? # simple numerical width i_max = np.argmax(hE) h_max = hE[i_max] upr_half = xE[(xE > xE[i_max]) & (hE <= h_max / 2)][0] bot_half = xE[(xE < xE[i_max]) & (hE >= h_max / 2)][0] fwhm = upr_half - bot_half sig = fwhm / 2.355 # fit to gaussian: amp, mu, sig, bkg fit_func = pgf.gauss_bkg amp = h_max * fwhm bg0 = np.mean(hE[:20]) x0 = [amp, xE[i_max], sig, bg0] xF, xF_cov = pgf.fit_hist(fit_func, hE, bins, var=vE, guess=x0) # collect results e_fit = xF[0] xF_err = np.sqrt(np.diag(xF_cov)) e_err = xF fwhm_fit = xF[1] * 2.355 * 1460. / e_fit fwhm_err = xF_err[2] * 2.355 * 1460. / e_fit chisq = [] for i, h in enumerate(hE): model = fit_func(xE[i], *xF) diff = (model - h)**2 / model chisq.append(abs(diff)) rchisq = sum(np.array(chisq) / len(hE)) fwhm_ovr_mean = fwhm_fit / e_fit if show_movie: plt.plot(xE, hE, ds='steps', c='b', lw=2, label=f'{etype} {rise}--{flat}') # peak shape plt.plot(xE, fit_func(xE, *x0), '-', c='orange', alpha=0.5, label='init. guess') plt.plot(xE, fit_func(xE, *xF), '-r', alpha=0.8, label='peakshape fit') plt.plot(np.nan, np.nan, '-w', label=f'mu={e_fit:.1f}, fwhm={fwhm_fit:.2f}') plt.xlabel(etype, ha='right', x=1) plt.ylabel('Counts', ha='right', y=1) plt.legend(loc=2) # show a little movie plt.show(block=False) plt.pause(0.01) plt.cla() # return results return pd.Series({ 'e_fit': e_fit, 'fwhm_fit': fwhm_fit, 'rchisq': rchisq, 'fwhm_err': xF_err[0], 'fwhm_ovr_mean': fwhm_ovr_mean }) # df_grid=df_grid[:10] df_tmp = df_grid.progress_apply(run_dsp, axis=1) df_grid[new_cols] = df_tmp # print(df_grid) if show_movie: plt.close() print('elapsed:', time.time() - t_start) if write_output: df_grid.to_hdf(f_results, key=grp_grid) print(f"Wrote output file: {f_results}")
# DCR calculation: use slope using 1000 samples apart and averaging 200 # samples, with the start 1.5 us offset from t0 proc.add_processor(trap_pickoff, "wf_pz", 200, 1000, "tp_0+1.5*us", "dcr_unnorm") proc.add_processor(np.divide, "dcr_unnorm", "trapEftp", "dcr") # Tail slope. Basically the same as DCR, except with no PZ correction proc.add_processor(linear_fit, "wf_blsub[3000:]", "wf_b", "wf_m") proc.add_processor(np.divide, "-wf_b", "wf_m", "tail_rc") # Set up the LH5 output lh5_out = lh5.Table(size=proc._buffer_len) lh5_out.add_field( "trapEmax", lh5.Array(proc.get_output_buffer("trapEmax"), attrs={"units": "ADC"})) lh5_out.add_field( "trapEftp", lh5.Array(proc.get_output_buffer("trapEftp"), attrs={"units": "ADC"})) lh5_out.add_field( "ct_corr", lh5.Array(proc.get_output_buffer("ct_corr"), attrs={"units": "ADC*ns"})) lh5_out.add_field( "bl", lh5.Array(proc.get_output_buffer("bl"), attrs={"units": "ADC"})) lh5_out.add_field( "bl_sig", lh5.Array(proc.get_output_buffer("bl_sig"), attrs={"units": "ADC"})) lh5_out.add_field( "A", lh5.Array(proc.get_output_buffer("curr_amp"), attrs={"units": "ADC"}))
def process_ds(f_grid, f_opt, f_tier1, d_out, efilter): """ process the windowed raw file 'f_tier1' and create the DSP file 'f_opt' """ print("Grid file:",f_grid) df_grid = pd.read_hdf(f_grid) if os.path.exists(f_opt): os.remove(f_opt) if 'corr' in efilter: bfilter = efilter.split('corr')[0] try: df_res = pd.read_hdf(f'{d_out}/{bfilter}_results.h5',key='results') print("Extraction of best parameters for", bfilter) except: print(bfilter,"not optimized") return # open raw file lh5_in = lh5.Store() #groups = lh5_in.ls(f_tier1, '*/raw') f = h5py.File(f_tier1,'r') #print("File info: ",f.keys()) t_start = time.time() #for group in groups: for idx, ged in enumerate(f.keys()): if idx == 4: diff = time.time() - t_start tot = diff/5 * len(df_grid) / 60 tot -= diff / 60 print(f"Estimated remaining time: {tot:.2f} mins") print("Detector:",ged) #data = lh5_in.read_object(group, f_tier1) data = f[ged]['raw'] #wf_in = data['waveform']['values'].nda #dt = data['waveform']['dt'].nda[0] * unit_parser.parse_unit(data['waveform']['dt'].attrs['units']) wf_in = data['waveform']['values'][()] dt = data['waveform']['dt'][0] * unit_parser.parse_unit(data['waveform']['dt'].attrs['units']) bl_in = data['baseline'][()] #flashcam baseline values # Set up DSP processing chain -- very minimal block = 8 #waveforms to process simultaneously proc = ProcessingChain(block_width=block, clock_unit=dt, verbosity=False) proc.add_input_buffer("wf", wf_in, dtype='float32') proc.add_input_buffer("bl", bl_in, dtype='float32') wsize = wf_in.shape[1] dt0 = data['waveform']['dt'][0]*0.001 #proc.add_processor(mean_stdev, "wf[0:1000]", "bl", "bl_sig") proc.add_processor(np.subtract, "wf", "bl", "wf_blsub") for i, row in df_grid.iterrows(): if 'corr' in efilter: ct_const = row if 'trapE' in efilter: if 'corr' in efilter: rise, flat, rc = float(df_res['rise'][idx]), float(df_res['flat'][idx]), float(df_res['rc'][idx]) else: rise, flat, rc = row proc.add_processor(pole_zero, "wf_blsub", rc*us, "wf_pz") proc.add_processor(trap_norm, "wf_pz", rise*us, flat*us, f"wf_trap_{i}") proc.add_processor(asymTrapFilter, "wf_pz", 0.05*us, 4*us, 4*us, "wf_atrap") proc.add_processor(time_point_thresh, "wf_pz", 0, "tp_0") proc.add_processor(np.amax, f"wf_trap_{i}", 1, f"trapE_{i}", signature='(n),()->()', types=['fi->f']) proc.add_processor(fixed_time_pickoff, f"wf_trap_{i}", f"tp_0+({rise*us}+{flat*us})", f"trapEftp_{i}") if 'zacE' in efilter: if 'corr' in efilter: sigma, flat, decay = float(df_res['sigma'][idx]), float(df_res['flat'][idx]), float(df_res['decay'][idx]) else: sigma, flat, decay = row proc.add_processor(zac_filter(wsize, sigma/dt0, flat/dt0, decay/dt0),"wf", f"wf_zac_{i}(101, f)") proc.add_processor(np.amax, f"wf_zac_{i}", 1, f"zacE_{i}", signature='(n),()->()', types=['fi->f']) if 'cuspE' in efilter: if 'corr' in efilter: sigma, flat, decay = float(df_res['sigma'][idx]), float(df_res['flat'][idx]), float(df_res['decay'][idx]) else: sigma, flat, decay = row proc.add_processor(cusp_filter(wsize, sigma/dt0, flat/dt0, decay/dt0),"wf_blsub", f"wf_cusp_{i}(101, f)") proc.add_processor(np.amax, f"wf_cusp_{i}", 1, f"cuspE_{i}", signature='(n),()->()', types=['fi->f']) if 'corr' in efilter: proc.add_processor(trap_pickoff, "wf_pz", 1.5*us, 0, "tp_0", "ct_corr") #proc.add_processor(trap_pickoff, "wf_pz", rise*us, flat*us, "tp_0", "ct_corr") proc.add_processor(np.multiply, ct_const, "ct_corr", f"ct_corr_cal_{i}") proc.add_processor(np.add, f"ct_corr_cal_{i}", f"{bfilter}_{i}", f"{efilter}_{i}") # Set up the LH5 output lh5_out = lh5.Table(size=proc._buffer_len) for i, row in df_grid.iterrows(): lh5_out.add_field(f"{efilter}_{i}", lh5.Array(proc.get_output_buffer(f"{efilter}_{i}"), attrs={"units":"ADC"})) print("Processing:\n",proc) proc.execute() #groupname = group[:group.rfind('/')+1]+"data" #groupname = df_key+"/"+group+"/data" groupname = ged+"/data" print("Writing to: " + f_opt + "/" + groupname) lh5_in.write_object(lh5_out, groupname, f_opt) print("") #list the datasets of the output file data_opt = lh5_in.ls(f_opt) #data_opt_0 = lh5_in.ls(f_opt,'opt_0/*') data_opt_0 = lh5_in.ls(f_opt,'g024/data/*') diff = time.time() - t_start print(f"Time to process: {diff:.2f} s")
def dsp_to_hit_cage(f_dsp, f_hit, dg, n_max=None, verbose=False, t_start=None): """ non-general placeholder for creating a pygama 'hit' file. uses pandas. for every file, apply: - energy calibration (peakfit results) - timestamp correction for a more general dsp_to_hit, maybe each function could be given in terms of an 'apply' on a dsp dataframe ... TODO: create entry config['rawe'] with list of energy pars to calibrate, as in energy_cal.py """ rawe = ['trapEmax'] # create initial 'hit' DataFrame from dsp data hit_store = lh5.Store() data = hit_store.read_object(dg.config['input_table'], f_dsp) df_hit = data.get_dataframe() # 1. get energy calibration for this run from peakfit cal_db = db.TinyDB(storage=MemoryStorage) with open(dg.config['ecaldb']) as f: raw_db = json.load(f) cal_db.storage.write(raw_db) runs = dg.file_keys.run.unique() if len(runs) > 1: print("sorry, I can't do combined runs yet") exit() run = runs[0] for etype in rawe: tb = cal_db.table(f'peakfit_{etype}').all() df_cal = pd.DataFrame(tb) df_cal['run'] = df_cal['run'].astype(int) df_run = df_cal.loc[df_cal.run == run] cal_pars = df_run.iloc[0][['cal0', 'cal1', 'cal2']] pol = np.poly1d(cal_pars) # handy numpy polynomial object df_hit[f'{etype}_cal'] = pol(df_hit[f'{etype}']) # 2. compute timestamp rollover correction (specific to struck 3302) clock = 100e6 # 100 MHz UINT_MAX = 4294967295 # (0xffffffff) t_max = UINT_MAX / clock ts = df_hit['timestamp'].values / clock tdiff = np.diff(ts) tdiff = np.insert(tdiff, 0, 0) iwrap = np.where(tdiff < 0) iloop = np.append(iwrap[0], len(ts)) ts_new, t_roll = [], 0 for i, idx in enumerate(iloop): ilo = 0 if i == 0 else iwrap[0][i - 1] ihi = idx ts_block = ts[ilo:ihi] t_last = ts[ilo - 1] t_diff = t_max - t_last ts_new.append(ts_block + t_roll) t_roll += t_last + t_diff df_hit['ts_sec'] = np.concatenate(ts_new) # 3. compute global timestamp if t_start is not None: df_hit['ts_glo'] = df_hit['ts_sec'] + t_start # write to LH5 file if os.path.exists(f_hit): os.remove(f_hit) sto = lh5.Store() tb_name = dg.config['input_table'].replace('dsp', 'hit') tb_lh5 = lh5.Table(size=len(df_hit)) for col in df_hit.columns: tb_lh5.add_field(col, lh5.Array(df_hit[col].values, attrs={'units': ''})) print(col) print(f'Writing table: {tb_name} in file:\n {f_hit}') sto.write_object(tb_lh5, tb_name, f_hit)
def raw_to_dsp(ds, overwrite=False, nevt=None, test=False, verbose=2, block=8, group='daqdata'): """ Run raw_to_dsp on a set of runs. [raw file] ---> [dsp_run{}.lh5] (digital signal processing results) """ for run in ds.runs: raw_file = ds.paths[run]["raw_path"] dsp_file = ds.paths[run]["dsp_path"] if dsp_file is not None and overwrite is False: continue if dsp_file is None: # declare new file name dsp_file = raw_file.replace('raw', 'dsp') if test: print("test mode (dry run), processing raw file:", raw_file) continue # new LH5 version lh5_in = lh5.Store() data = lh5_in.read_object("/ORSIS3302DecoderForEnergy", raw_file) wf_in = data['waveform']['values'].nda dt = data['waveform']['dt'].nda[0] * unit_parser.parse_unit( data['waveform']['dt'].attrs['units']) # Parameters for DCR calculation dcr_trap_int = 200 dcr_trap_flat = 1000 dcr_trap_startSample = 1200 # Set up processing chain proc = ProcessingChain(block_width=block, clock_unit=dt, verbosity=verbose) proc.add_input_buffer("wf", wf_in, dtype='float32') proc.add_processor(mean_stdev, "wf[0:1000]", "bl", "bl_sig") proc.add_processor(np.subtract, "wf", "bl", "wf_blsub") proc.add_processor(pole_zero, "wf_blsub", 70 * us, "wf_pz") proc.add_processor(asymTrapFilter, "wf_pz", 10 * us, 5 * us, 10 * us, "wf_atrap") proc.add_processor(np.amax, "wf_atrap", 1, "atrapE", signature='(n),()->()', types=['fi->f']) # proc.add_processor(np.divide, "atrapmax", 10*us, "atrapE") proc.add_processor(trap_norm, "wf_pz", 10 * us, 5 * us, "wf_trap") proc.add_processor(np.amax, "wf_trap", 1, "trapE", signature='(n),()->()', types=['fi->f']) proc.add_processor(avg_current, "wf_pz", 10, "curr") proc.add_processor(np.amax, "curr", 1, "A_10", signature='(n),()->()', types=['fi->f']) proc.add_processor(np.divide, "A_10", "trapE", "AoE") proc.add_processor(trap_pickoff, "wf_pz", dcr_trap_int, dcr_trap_flat, dcr_trap_startSample, "dcr") # Set up the LH5 output lh5_out = lh5.Table(size=proc._buffer_len) lh5_out.add_field( "trapE", lh5.Array(proc.get_output_buffer("trapE"), attrs={"units": "ADC"})) lh5_out.add_field( "bl", lh5.Array(proc.get_output_buffer("bl"), attrs={"units": "ADC"})) lh5_out.add_field( "bl_sig", lh5.Array(proc.get_output_buffer("bl_sig"), attrs={"units": "ADC"})) lh5_out.add_field( "A", lh5.Array(proc.get_output_buffer("A_10"), attrs={"units": "ADC"})) lh5_out.add_field( "AoE", lh5.Array(proc.get_output_buffer("AoE"), attrs={"units": "ADC"})) lh5_out.add_field( "dcr", lh5.Array(proc.get_output_buffer("dcr"), attrs={"units": "ADC"})) print("Processing:\n", proc) proc.execute() print("Writing to: ", dsp_file) f_lh5.write_object(lh5_out, "data", dsp_file)
def build_processing_chain(lh5_in, dsp_config, outputs=None, verbosity=1, block_width=8): """ Produces a ProcessingChain object and an lh5 table for output parameters from an input lh5 table and a json recipe. Returns (proc_chain, lh5_out): - proc_chain: ProcessingChain object that is bound to lh5_in and lh5_out; all you need to do is handle file i/o for lh5_in/out and run execute - lh5_out: output LH5 table Required arguments: - lh5_in: input LH5 table - config: dict or name of json file containing a recipe for constructing the ProcessingChain object produced by this function. config is formated as a json dict with different processors. Config should have a dictionary called processors, containing dictionaries of the following format: Key: parameter name: name of parameter produced by the processor. can optionally provide multiple, separated by spaces Values: processor (req): name of gufunc module (req): name of module in which to find processor prereqs (req): name of parameters from other processors and from input that are required to exist to run this args (req): list of arguments for processor, with variables passed by name or value. Names should either be inputs from lh5_in, or parameter names for other processors. Names of the format db.name will look up the parameter in the metadata. kwargs (opt): kwargs used when adding processors to proc_chain init_args (opt): args used when initializing a processor that has static data (for factory functions) default (opt): default value for db parameters if not found unit (opt): unit to be used for attr in lh5 file. There may also be a list called 'outputs', containing a list of parameters to put into lh5_out. Optional keyword arguments: - outputs: list of parameters to put in the output lh5 table. If None, use the parameters in the 'outputs' list from config - verbosity: verbosity level: 0: Print nothing (except errors...) 1: Print basic warnings (default) 2: Print basic debug info 3: Print friggin' everything! - block_width: number of entries to process at once. """ if isinstance(dsp_config, str): with open(dsp_config) as f: dsp_config = json.load(f) elif dsp_config is None: dsp_config = {'outputs': [], 'processors': {}} else: # We don't want to modify the input! dsp_config = deepcopy(dsp_config) if outputs is None: outputs = dsp_config['outputs'] processors = dsp_config['processors'] # for processors with multiple outputs, add separate entries to the processor list for key in list(processors): keys = [k for k in re.split(",| ", key) if k != ''] if len(keys) > 1: for k in keys: processors[k] = key # Recursive function to crawl through the parameters/processors and get # a sequence of unique parameters such that parameters always appear after # their dependencies. For parameters that are not produced by the ProcChain # (i.e. input/db parameters), add them to the list of leafs # https://www.electricmonk.nl/docs/dependency_resolving_algorithm/dependency_resolving_algorithm.html def resolve_dependencies(par, resolved, leafs, unresolved=[]): if par in resolved: return elif par in unresolved: raise Exception('Circular references detected: %s -> %s' % (par, edge)) # if we don't find a node, this is a leaf node = processors.get(par) if node is None: if par not in leafs: leafs.append(par) return # if it's a string, that means it is part of a processor that returns multiple outputs (see above); in that case, node is a str pointing to the actual node we want if isinstance(node, str): resolve_dependencies(node, resolved, leafs, unresolved) return edges = node['prereqs'] unresolved.append(par) for edge in edges: resolve_dependencies(edge, resolved, leafs, unresolved) resolved.append(par) unresolved.remove(par) proc_par_list = [] # calculated from processors input_par_list = [] # input from file and used for processors copy_par_list = [] # copied from input to output out_par_list = [] for out_par in outputs: if out_par not in processors: copy_par_list.append(out_par) else: resolve_dependencies(out_par, proc_par_list, input_par_list) out_par_list.append(out_par) if verbosity > 0: print('Processing parameters:', str(proc_par_list)) print('Required input parameters:', str(input_par_list)) print('Copied output parameters:', str(copy_par_list)) print('Processed output parameters:', str(out_par_list)) proc_chain = ProcessingChain(block_width, lh5_in.size, verbosity=verbosity) # Now add all of the input buffers from lh5_in (and also the clk time) for input_par in input_par_list: buf_in = lh5_in.get(input_par) if buf_in is None: print("I don't know what to do with " + input_par + ". Building output without it!") elif isinstance(buf_in, lh5.Array): proc_chain.add_input_buffer(input_par, buf_in.nda) elif isinstance(buf_in, lh5.Table): # check if this is waveform if 't0' and 'dt' and 'values' in buf_in: proc_chain.add_input_buffer(input_par, buf_in['values'].nda, 'float32') clk = buf_in['dt'].nda[0] * unit_parser.parse_unit( lh5_in['waveform']['dt'].attrs['units']) if proc_chain._clk is not None and proc_chain._clk != clk: print( "Somehow you managed to set multiple clock frequencies...Using " + str(proc_chain._clk)) else: proc_chain._clk = clk # now add the processors for proc_par in proc_par_list: recipe = processors[proc_par] module = importlib.import_module(recipe['module']) func = getattr(module, recipe['function']) args = recipe['args'] for i, arg in enumerate(args): if isinstance(arg, str) and arg[0:3] == 'db.': #TODO: ADD METADATA LOOKUP! args[i] = recipe['defaults'][arg] kwargs = recipe.get('kwargs', {}) # might also need metadata lookup here # if init_args are defined, parse any strings and then call func # as a factory/constructor function try: init_args = recipe['init_args'] for i, arg in enumerate(init_args): if isinstance(arg, str): if arg[0:3] == 'db.': #TODO: ADD METADATA LOOKUP! init_args[i] = recipe['defaults'][arg] else: # see if string can be parsed by proc_chain try: init_args[i] = proc_chain.get_variable(arg) except: pass if (verbosity > 1): print("Building function", func.__name__, "from init_args", init_args) func = func(*init_args) except: pass proc_chain.add_processor(func, *args, **kwargs) # build the output buffers lh5_out = lh5.Table(size=proc_chain._buffer_len) # add inputs that are directly copied for copy_par in copy_par_list: buf_in = lh5_in.get(copy_par) if isinstance(buf_in, lh5.Array): lh5_out.add_field(copy_par, buf_in) elif isinstance(buf_in, lh5.Table): # check if this is waveform if 't0' and 'dt' and 'values' in buf_in: lh5_out.add_field(copy_par, buf_in['values']) clk = buf_in['dt'].nda[0] * unit_parser.parse_unit( lh5_in['waveform']['dt'].attrs['units']) if proc_chain._clk is not None and proc_chain._clk != clk: print( "Somehow you managed to set multiple clock frequencies...Using " + str(proc_chain._clk)) else: proc_chain._clk = clk else: print("I don't know what to do with " + input_par + ". Building output without it!") # finally, add the output buffers to lh5_out and the proc chain for out_par in out_par_list: recipe = processors[out_par] # special case for proc with multiple outputs if isinstance(recipe, str): i = [k for k in re.split(",| ", recipe) if k != ''].index(out_par) recipe = processors[recipe] unit = recipe['unit'][i] else: unit = recipe['unit'] try: scale = convert(1, unit_parser.parse_unit(unit), clk) except InvalidConversion: scale = None buf_out = proc_chain.get_output_buffer(out_par, unit=scale) lh5_out.add_field(out_par, lh5.Array(buf_out, attrs={"units": unit})) return (proc_chain, lh5_out)
def load_raw_data_example(f_raw): """ make a plot of the timestamps in a particular channel. instead of accessing just the timestamp column, this is an example of accessing the entire raw file (including waveforms) with LH5. """ sto = lh5.Store() tb_name = 'g024/raw' n_rows = 100 # np.inf to read all # method 1: call load_nda to pull out only timestamp column (fast) # par_data = lh5.load_nda([f_raw], ['timestamp'], tb_name) # pprint(par_data) # print(par_data['timestamp'].shape) # exit() # method 2: read all data, just to give a longer example of what we can access # TODO: include an example of slicing/selecting rows with np.where # read non-wf cols (lh5 Arrays) data_raw, n_tot = sto.read_object(tb_name, f_raw, n_rows=n_rows) # declare output table (must specify n_rows for size) tb_raw = lh5.Table(size=n_tot) for col in data_raw.keys(): if col in ['waveform', 'tracelist']: continue # copy all values newcol = lh5.Array(data_raw[col].nda, attrs=data_raw[col].attrs) # copy a selection (using np.where) # newcol = lh5.Array(data_raw[col].nda[idx], attrs=data_raw[col].attrs) tb_raw.add_field(col, newcol) df_raw = tb_raw.get_dataframe() print(df_raw) # load waveform column (nested LH5 Table) data_wfs, n_tot = sto.read_object(tb_name + '/waveform', f_raw, n_rows=n_rows) tb_wfs = lh5.Table(size=n_tot) for col in data_wfs.keys(): attrs = data_wfs[col].attrs if isinstance(data_wfs[col], lh5.ArrayOfEqualSizedArrays): # idk why i can't put the filtered array into the constructor aoesa = lh5.ArrayOfEqualSizedArrays(attrs=attrs, dims=[1, 1]) aoesa.nda = data_wfs[col].nda # aoesa.nda = data_wfs[col].nda[idx] # with np.where selection newcol = aoesa else: newcol = lh5.Array(data_wfs[col].nda, attrs=attrs) # newcol = lh5.Array(data_wfs[col].nda[idx], attrs=attrs) # selection tb_wfs.add_field(col, newcol) tb_wfs.add_field('waveform', newcol) tb_wfs.attrs = data_raw.attrs # can write to file, to read back in for DSP, etc. # sto.write_object(tb_raw, grp_data, f_peak) print(tb_wfs) print(tb_wfs['waveform'].shape)