def d2r(dg, overwrite=False, nwfs=None, vrb=False): """ run daq_to_raw on the current DataGroup """ # print(dg.file_keys) # print(dg.file_keys.columns) subs = dg.subsystems # can be blank: [''] # subs = ['geds'] # TODO: ignore other datastreams # chans = ['g035', 'g042'] # TODO: select a subset of detectors print(f'Processing {dg.file_keys.shape[0]} files ...') for i, row in dg.file_keys.iterrows(): f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}" f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" # f_raw = 'test.lh5' subrun = row['cycle'] if 'cycle' in row else None if not overwrite and os.path.exists(f_raw): print('file exists, overwrite not set, skipping f_raw:\n ', f_raw) continue daq_to_raw(f_daq, f_raw, config=dg.config, systems=subs, verbose=vrb, n_max=nwfs, overwrite=overwrite, subrun=subrun) #, chans=chans)
def daq_to_raw(ds, overwrite=False, nevt=np.inf, v=False, test=False): """ Run daq_to_raw on a set of runs. [raw file] ---> [raw_run{}.lh5] (basic info & waveforms) """ from pygama.io.daq_to_raw import daq_to_raw for run in ds.runs: daq_file = "/lfs/l1/legend/users/dandrea/pygama/pgt/raw/pgt_longtrace_run0117-20200110-105115-calib.fcio" raw_file = "pgt_longtrace_run0117-20200110-105115-calib_raw.lh5" #daq_file = ds.paths[run]["daq_path"] #raw_file = ds.paths[run]["raw_path"] if raw_file is not None and overwrite is False: print("file exists, overwrite flag isn't set. continuing ...") continue #conf = ds.paths[run]["build_opt"] #opts = ds.config["build_options"][conf]["daq_to_raw_options"] opts = ds.config["daq_to_raw"] print("opts",opts) if test: print("test mode (dry run), processing DAQ file:", daq_file) print("output file:", raw_file) continue # # old pandas version # daq_to_raw(daq_file, run, verbose=v, output_dir=ds.raw_dir, # overwrite=overwrite, n_max=nevt, config=ds.config)#settings=opts) # new lh5 version daq_to_raw(daq_file, raw_filename=raw_file, run=run, chan_list=None, prefix=ds.rawpre, n_max=nevt, verbose=v, output_dir=ds.raw_dir, overwrite=overwrite, config=ds.config)
def daq_to_raw(self, overwrite=False, test=False, n_max=np.inf): """ convenience function for calling the main daq_to_raw function. right now, this processes runs sequentially. """ from pygama.io.daq_to_raw import daq_to_raw for run in self.runs: t0_file = self.paths[run]["t0_path"] t1_file = self.paths[run]["t1_path"] if t1_file is not None and overwrite is False: print("file exists, overwrite flag isn't set. continuing ...") continue if test: print("test mode (dry run), processing Tier 0 file:\n ", t0_file) continue daq_to_raw(t0_file, run, suffix=self.config["suffix"], verbose=test, output_dir=self.tier1_dir, overwrite=overwrite, n_max=n_max, config=self.config)
def d2r(ds, overwrite=False, nevt=np.inf, v=False, test=False): """ Run daq_to_raw on a set of runs. [raw file] ---> [t1_run{}.h5] (tier 1 file: basic info & waveforms) """ from pygama.io.daq_to_raw import daq_to_raw for run in ds.runs: t0_file = ds.paths[run]["t0_path"] t1_file = ds.paths[run]["t1_path"] if t1_file is not None and overwrite is False: print("file exists, overwrite flag isn't set. continuing ...") continue conf = ds.paths[run]["build_opt"] opts = ds.config["build_options"][conf]["daq_to_raw_options"] if test: print("test mode (dry run), processing Tier 0 file:", t0_file) continue daq_to_raw(t0_file, run, verbose=v, output_dir=ds.tier1_dir, overwrite=overwrite, n_max=nevt, config=ds.config) #, settings=opts)
def d2r(dg, overwrite=False, nwfs=None, vrb=False): """ run daq_to_raw on the current DataGroup """ # print(dg.file_keys) # print(dg.file_keys.columns) # subs = dg.subsystems # can be blank: [''] # subs = ['geds'] # TODO: ignore other datastreams # chans = ['g035', 'g042'] # TODO: select a subset of detectors print(f'Processing {dg.file_keys.shape[0]} files ...') for i, row in dg.file_keys.iterrows(): if row['cycle'] < 44: print('skipping') else: # set up I/O paths f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}" f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" subrun = row['cycle'] if 'cycle' in row else None systems = dg.subsystems daq_to_raw(f_daq, f_raw, config=dg.config, verbose=vrb, n_max=nwfs, overwrite=overwrite, subrun=subrun, systems=systems) #, chans=chans)
def main(argv): """ Uses pygama's amazing DataSet class to process runs for different data sets and arbitrary configuration options defined in a JSON file. """ #datadir = os.environ["CAGEDATA"] run_db, cal_db = f'./meta/runDB.json', f'./meta/calDB.json' # -- parse args -- par = argparse.ArgumentParser(description="data processing suite for CAGE") arg, st, sf = par.add_argument, "store_true", "store_false" arg("-ds", nargs='*', action="store", help="load runs for a DS") arg("-r", "--run", nargs=1, help="load a single run") arg("-d2r", "--daq_to_raw", action=st, help="run daq_to_raw on list") arg("-r2d", "--raw_to_dsp", action=st, help="run raw_to_dsp on list") arg("-t", "--test", action=st, help="test mode, don't run") arg("-n", "--nevt", nargs='?', default=np.inf, help="limit max num events") arg("-i", "--ioff", nargs='?', default=0, help="start at index [i]") arg("-o", "--ovr", action=st, help="overwrite existing files") arg('-v', '--verbose', default=2, type=int, help="Verbosity level: 0=silent, 1=basic warnings, 2=verbose output, 3=debug. Default is 2.") arg('-b', '--block', default=8, type=int, help="Number of waveforms to process simultaneously. Default is 8") arg('-g', '--group', default='', help="Name of group in LH5 file. By default process all base groups. Supports wildcards.") # -- declare the DataSet -- args = par.parse_args() d_args = vars(par.parse_args()) #ds = pu.get_dataset_from_cmdline(d_args, run_db, cal_db) # -- declare the DataSet -- if d_args["ds"]: ds_lo = int(d_args["ds"][0]) try: ds_hi = int(d_args["ds"][1]) except: ds_hi = None ds = DataSet(1,ds_lo, ds_hi, md=run_db, v=d_args["verbose"]) if d_args["run"]: ds = DataSet(1,run=int(d_args["run"][0]), md=run_db, v=d_args["verbose"]) #print(ds.runs) #pprint(ds.paths) # -- start processing -- if args.daq_to_raw: daq_to_raw(ds, args.ovr, args.nevt, args.verbose, args.test) if args.raw_to_dsp: raw_to_dsp(ds, args.ovr, args.nevt, args.test, args.verbose, args.block, args.group)
def d2r(dg, overwrite=False, nwfs=None, vrb=False, cwd=False): """ run daq_to_raw on the current DataGroup """ # print(dg.fileDB) # print(dg.fileDB.columns) # subs = ['geds'] # TODO: ignore other datastreams # chans = ['g035', 'g042'] # TODO: select a subset of detectors print(f'Processing {dg.fileDB.shape[0]} files ...') for i, row in dg.fileDB.iterrows(): # set up I/O paths f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}" f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" if cwd: f_raw = f"{row['raw_file']}" subrun = row['cycle'] if 'cycle' in row else None systems = dg.subsystems # load cmap if there is one if 'cmap' in row: cmap = row['cmap'] if ('daq_to_raw' in dg.config and 'ch_groups' in dg.config['daq_to_raw'] and 'FlashCamEventDecoder' in dg.config['daq_to_raw']['ch_groups']): # we are going to overwrite the "default" in config with # the cmap, so give a nice warning print( 'overwriting default ch_groups in config file with channel map info in', cmap) else: if 'daq_to_raw' not in dg.config: dg.config['daq_to_raw'] = {} if 'ch_groups' not in dg.config['daq_to_raw']: dg.config['daq_to_raw']['ch_groups'] = {} ch_groups = cmap_to_ch_groups(cmap) dg.config['daq_to_raw']['ch_groups'][ 'FlashCamEventDecoder'] = ch_groups systems = get_list_of('system', ch_groups) systems.append('auxs') # for FC status info daq_to_raw(f_daq, f_raw, config=dg.config, systems=systems, verbose=vrb, n_max=nwfs, overwrite=overwrite, subrun=subrun) #, chans=chans)
def d2r(dg, overwrite=False, nwfs=None, verbose=False, user=False): """ $ ./processing.py -q 'run==[something]' --d2r run daq_to_raw on the current DataGroup """ # print(dg.fileDB) # print(dg.fileDB.columns) subs = dg.subsystems # can be blank: [''] # subs = ['geds'] # TODO: ignore other datastreams # chans = ['g035', 'g042'] # TODO: select a subset of detectors print(f'Processing {dg.fileDB.shape[0]} files ...') for i, row in dg.fileDB.iterrows(): lh5_dir = dg.lh5_user_dir if user else dg.lh5_dir f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}" f_raw = f"{lh5_dir}/{row['raw_path']}/{row['raw_file']}" # f_raw = 'test.lh5' subrun = row['cycle'] if 'cycle' in row else None if pd.isna(row['raw_path']) or pd.isna(row['raw_file']): print('Error, nans in filenames/paths. Check your fileDB.') print('f_daq:', f_daq) print('f_raw:', f_raw) print(row) exit() if not overwrite and os.path.exists(f_raw): print('file exists, overwrite not set, skipping f_raw:\n ', f_raw) continue cyc = row['cycle'] if row.skip: print(f'Cycle {cyc} has been marked junk, will not process.') continue print(f'Processing cycle {cyc}') daq_to_raw(f_daq, f_raw, config=dg.config, systems=subs, verbose=verbose, n_max=nwfs, overwrite=overwrite, subrun=subrun) #, chans=chans)
def main(argv): """ Uses pygama's amazing DataSet class to process runs for different data sets and arbitrary configuration options defined in a JSON file. """ run_db = './runDB.json' # -- parse args -- par = argparse.ArgumentParser(description="data processing suite for MJ60") arg, st, sf = par.add_argument, "store_true", "store_false" arg("-ds", nargs='*', action="store", help="load runs for a DS") arg("-r", "--run", nargs=1, help="load a single run") arg("-t0", "--daq_to_raw", action=st, help="run ProcessRaw on list") arg("-t1", "--raw_to_dsp", action=st, help="run RunDSP on list") arg("-t", "--test", action=st, help="test mode, don't run") arg("-n", "--nevt", nargs='?', default=np.inf, help="limit max num events") arg("-i", "--ioff", nargs='?', default=0, help="start at index [i]") arg("-v", "--verbose", action=st, help="set verbose output") arg("-o", "--ovr", action=st, help="overwrite existing files") arg("-m", "--nomp", action=sf, help="don't use multiprocessing") args = vars(par.parse_args()) # -- declare the DataSet -- if args["ds"]: ds_lo = int(args["ds"][0]) try: ds_hi = int(args["ds"][1]) except: ds_hi = None ds = DataSet(ds_lo, ds_hi, md=run_db, v=args["verbose"]) if args["run"]: ds = DataSet(run=int(args["run"][0]), md=run_db, v=args["verbose"]) # -- start processing -- if args["daq_to_raw"]: daq_to_raw(ds, args["ovr"], args["nevt"], args["verbose"], args["test"]) if args["raw_to_dsp"]: raw_to_dsp(ds, args["ovr"], args["nevt"], args["ioff"], args["nomp"], args["verbose"], args["test"])
def daq_to_raw(self, overwrite=False, test=False, n_max=np.inf): """ TODO: duplicate the CAGE/processing code here, so that someone can run from command line: convenience function for calling the main daq_to_raw function. right now, this processes runs sequentially. """ from pygama.io.daq_to_raw import daq_to_raw for run in self.runs: daq_file = self.paths[run]["daq_path"] raw_file = self.paths[run]["raw_path"] if raw_file is not None and overwrite is False: print("file exists, overwrite flag isn't set. continuing ...") continue if test: print("test mode (dry run), processing Tier 0 file:\n ", daq_file) continue daq_to_raw(daq_file, run, suffix=self.config["suffix"], verbose=test, output_dir=self.raw_dir, overwrite=overwrite, n_max=n_max, config=self.config)