def d2r(dg, overwrite=False, nwfs=None, vrb=False):
"""
run daq_to_raw on the current DataGroup
"""
# print(dg.file_keys)
# print(dg.file_keys.columns)
subs = dg.subsystems # can be blank: ['']
# subs = ['geds'] # TODO: ignore other datastreams
# chans = ['g035', 'g042'] # TODO: select a subset of detectors
print(f'Processing {dg.file_keys.shape[0]} files ...')
for i, row in dg.file_keys.iterrows():
f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}"
f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}"
# f_raw = 'test.lh5'
subrun = row['cycle'] if 'cycle' in row else None
if not overwrite and os.path.exists(f_raw):
print('file exists, overwrite not set, skipping f_raw:\n ',
f_raw)
continue
daq_to_raw(f_daq,
f_raw,
config=dg.config,
systems=subs,
verbose=vrb,
n_max=nwfs,
overwrite=overwrite,
subrun=subrun) #, chans=chans)
def daq_to_raw(ds, overwrite=False, nevt=np.inf, v=False, test=False):
"""
Run daq_to_raw on a set of runs.
[raw file] ---> [raw_run{}.lh5] (basic info & waveforms)
"""
from pygama.io.daq_to_raw import daq_to_raw
for run in ds.runs:
daq_file = "/lfs/l1/legend/users/dandrea/pygama/pgt/raw/pgt_longtrace_run0117-20200110-105115-calib.fcio"
raw_file = "pgt_longtrace_run0117-20200110-105115-calib_raw.lh5"
#daq_file = ds.paths[run]["daq_path"]
#raw_file = ds.paths[run]["raw_path"]
if raw_file is not None and overwrite is False:
print("file exists, overwrite flag isn't set. continuing ...")
continue
#conf = ds.paths[run]["build_opt"]
#opts = ds.config["build_options"][conf]["daq_to_raw_options"]
opts = ds.config["daq_to_raw"]
print("opts",opts)
if test:
print("test mode (dry run), processing DAQ file:", daq_file)
print("output file:", raw_file)
continue
# # old pandas version
# daq_to_raw(daq_file, run, verbose=v, output_dir=ds.raw_dir,
# overwrite=overwrite, n_max=nevt, config=ds.config)#settings=opts)
# new lh5 version
daq_to_raw(daq_file, raw_filename=raw_file, run=run, chan_list=None,
prefix=ds.rawpre, n_max=nevt, verbose=v, output_dir=ds.raw_dir,
overwrite=overwrite, config=ds.config)
def daq_to_raw(self, overwrite=False, test=False, n_max=np.inf):
"""
convenience function for calling the main daq_to_raw function.
right now, this processes runs sequentially.
"""
from pygama.io.daq_to_raw import daq_to_raw
for run in self.runs:
t0_file = self.paths[run]["t0_path"]
t1_file = self.paths[run]["t1_path"]
if t1_file is not None and overwrite is False:
print("file exists, overwrite flag isn't set. continuing ...")
continue
if test:
print("test mode (dry run), processing Tier 0 file:\n ",
t0_file)
continue
daq_to_raw(t0_file,
run,
suffix=self.config["suffix"],
verbose=test,
output_dir=self.tier1_dir,
overwrite=overwrite,
n_max=n_max,
config=self.config)
def d2r(ds, overwrite=False, nevt=np.inf, v=False, test=False):
"""
Run daq_to_raw on a set of runs.
[raw file] ---> [t1_run{}.h5] (tier 1 file: basic info & waveforms)
"""
from pygama.io.daq_to_raw import daq_to_raw
for run in ds.runs:
t0_file = ds.paths[run]["t0_path"]
t1_file = ds.paths[run]["t1_path"]
if t1_file is not None and overwrite is False:
print("file exists, overwrite flag isn't set. continuing ...")
continue
conf = ds.paths[run]["build_opt"]
opts = ds.config["build_options"][conf]["daq_to_raw_options"]
if test:
print("test mode (dry run), processing Tier 0 file:", t0_file)
continue
daq_to_raw(t0_file,
run,
verbose=v,
output_dir=ds.tier1_dir,
overwrite=overwrite,
n_max=nevt,
config=ds.config) #, settings=opts)
def d2r(dg, overwrite=False, nwfs=None, vrb=False):
"""
run daq_to_raw on the current DataGroup
"""
# print(dg.file_keys)
# print(dg.file_keys.columns)
# subs = dg.subsystems # can be blank: ['']
# subs = ['geds'] # TODO: ignore other datastreams
# chans = ['g035', 'g042'] # TODO: select a subset of detectors
print(f'Processing {dg.file_keys.shape[0]} files ...')
for i, row in dg.file_keys.iterrows():
if row['cycle'] < 44:
print('skipping')
else:
# set up I/O paths
f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}"
f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}"
subrun = row['cycle'] if 'cycle' in row else None
systems = dg.subsystems
daq_to_raw(f_daq,
f_raw,
config=dg.config,
verbose=vrb,
n_max=nwfs,
overwrite=overwrite,
subrun=subrun,
systems=systems) #, chans=chans)
def main(argv):
"""
Uses pygama's amazing DataSet class to process runs
for different data sets and arbitrary configuration options
defined in a JSON file.
"""
#datadir = os.environ["CAGEDATA"]
run_db, cal_db = f'./meta/runDB.json', f'./meta/calDB.json'
# -- parse args --
par = argparse.ArgumentParser(description="data processing suite for CAGE")
arg, st, sf = par.add_argument, "store_true", "store_false"
arg("-ds", nargs='*', action="store", help="load runs for a DS")
arg("-r", "--run", nargs=1, help="load a single run")
arg("-d2r", "--daq_to_raw", action=st, help="run daq_to_raw on list")
arg("-r2d", "--raw_to_dsp", action=st, help="run raw_to_dsp on list")
arg("-t", "--test", action=st, help="test mode, don't run")
arg("-n", "--nevt", nargs='?', default=np.inf, help="limit max num events")
arg("-i", "--ioff", nargs='?', default=0, help="start at index [i]")
arg("-o", "--ovr", action=st, help="overwrite existing files")
arg('-v', '--verbose', default=2, type=int,
help="Verbosity level: 0=silent, 1=basic warnings, 2=verbose output, 3=debug. Default is 2.")
arg('-b', '--block', default=8, type=int,
help="Number of waveforms to process simultaneously. Default is 8")
arg('-g', '--group', default='',
help="Name of group in LH5 file. By default process all base groups. Supports wildcards.")
# -- declare the DataSet --
args = par.parse_args()
d_args = vars(par.parse_args())
#ds = pu.get_dataset_from_cmdline(d_args, run_db, cal_db)
# -- declare the DataSet --
if d_args["ds"]:
ds_lo = int(d_args["ds"][0])
try:
ds_hi = int(d_args["ds"][1])
except:
ds_hi = None
ds = DataSet(1,ds_lo, ds_hi, md=run_db, v=d_args["verbose"])
if d_args["run"]:
ds = DataSet(1,run=int(d_args["run"][0]), md=run_db, v=d_args["verbose"])
#print(ds.runs)
#pprint(ds.paths)
# -- start processing --
if args.daq_to_raw:
daq_to_raw(ds, args.ovr, args.nevt, args.verbose, args.test)
if args.raw_to_dsp:
raw_to_dsp(ds, args.ovr, args.nevt, args.test, args.verbose, args.block,
args.group)
def d2r(dg, overwrite=False, nwfs=None, vrb=False, cwd=False):
"""
run daq_to_raw on the current DataGroup
"""
# print(dg.fileDB)
# print(dg.fileDB.columns)
# subs = ['geds'] # TODO: ignore other datastreams
# chans = ['g035', 'g042'] # TODO: select a subset of detectors
print(f'Processing {dg.fileDB.shape[0]} files ...')
for i, row in dg.fileDB.iterrows():
# set up I/O paths
f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}"
f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}"
if cwd: f_raw = f"{row['raw_file']}"
subrun = row['cycle'] if 'cycle' in row else None
systems = dg.subsystems
# load cmap if there is one
if 'cmap' in row:
cmap = row['cmap']
if ('daq_to_raw' in dg.config
and 'ch_groups' in dg.config['daq_to_raw']
and 'FlashCamEventDecoder'
in dg.config['daq_to_raw']['ch_groups']):
# we are going to overwrite the "default" in config with
# the cmap, so give a nice warning
print(
'overwriting default ch_groups in config file with channel map info in',
cmap)
else:
if 'daq_to_raw' not in dg.config:
dg.config['daq_to_raw'] = {}
if 'ch_groups' not in dg.config['daq_to_raw']:
dg.config['daq_to_raw']['ch_groups'] = {}
ch_groups = cmap_to_ch_groups(cmap)
dg.config['daq_to_raw']['ch_groups'][
'FlashCamEventDecoder'] = ch_groups
systems = get_list_of('system', ch_groups)
systems.append('auxs') # for FC status info
daq_to_raw(f_daq,
f_raw,
config=dg.config,
systems=systems,
verbose=vrb,
n_max=nwfs,
overwrite=overwrite,
subrun=subrun) #, chans=chans)
def d2r(dg, overwrite=False, nwfs=None, verbose=False, user=False):
"""
$ ./processing.py -q 'run==[something]' --d2r
run daq_to_raw on the current DataGroup
"""
# print(dg.fileDB)
# print(dg.fileDB.columns)
subs = dg.subsystems # can be blank: ['']
# subs = ['geds'] # TODO: ignore other datastreams
# chans = ['g035', 'g042'] # TODO: select a subset of detectors
print(f'Processing {dg.fileDB.shape[0]} files ...')
for i, row in dg.fileDB.iterrows():
lh5_dir = dg.lh5_user_dir if user else dg.lh5_dir
f_daq = f"{dg.daq_dir}/{row['daq_dir']}/{row['daq_file']}"
f_raw = f"{lh5_dir}/{row['raw_path']}/{row['raw_file']}"
# f_raw = 'test.lh5'
subrun = row['cycle'] if 'cycle' in row else None
if pd.isna(row['raw_path']) or pd.isna(row['raw_file']):
print('Error, nans in filenames/paths. Check your fileDB.')
print('f_daq:', f_daq)
print('f_raw:', f_raw)
print(row)
exit()
if not overwrite and os.path.exists(f_raw):
print('file exists, overwrite not set, skipping f_raw:\n ',
f_raw)
continue
cyc = row['cycle']
if row.skip:
print(f'Cycle {cyc} has been marked junk, will not process.')
continue
print(f'Processing cycle {cyc}')
daq_to_raw(f_daq,
f_raw,
config=dg.config,
systems=subs,
verbose=verbose,
n_max=nwfs,
overwrite=overwrite,
subrun=subrun) #, chans=chans)
def main(argv):
"""
Uses pygama's amazing DataSet class to process runs
for different data sets and arbitrary configuration options
defined in a JSON file.
"""
run_db = './runDB.json'
# -- parse args --
par = argparse.ArgumentParser(description="data processing suite for MJ60")
arg, st, sf = par.add_argument, "store_true", "store_false"
arg("-ds", nargs='*', action="store", help="load runs for a DS")
arg("-r", "--run", nargs=1, help="load a single run")
arg("-t0", "--daq_to_raw", action=st, help="run ProcessRaw on list")
arg("-t1", "--raw_to_dsp", action=st, help="run RunDSP on list")
arg("-t", "--test", action=st, help="test mode, don't run")
arg("-n", "--nevt", nargs='?', default=np.inf, help="limit max num events")
arg("-i", "--ioff", nargs='?', default=0, help="start at index [i]")
arg("-v", "--verbose", action=st, help="set verbose output")
arg("-o", "--ovr", action=st, help="overwrite existing files")
arg("-m", "--nomp", action=sf, help="don't use multiprocessing")
args = vars(par.parse_args())
# -- declare the DataSet --
if args["ds"]:
ds_lo = int(args["ds"][0])
try:
ds_hi = int(args["ds"][1])
except:
ds_hi = None
ds = DataSet(ds_lo, ds_hi, md=run_db, v=args["verbose"])
if args["run"]:
ds = DataSet(run=int(args["run"][0]), md=run_db, v=args["verbose"])
# -- start processing --
if args["daq_to_raw"]:
daq_to_raw(ds, args["ovr"], args["nevt"], args["verbose"],
args["test"])
if args["raw_to_dsp"]:
raw_to_dsp(ds, args["ovr"], args["nevt"], args["ioff"], args["nomp"],
args["verbose"], args["test"])
def daq_to_raw(self, overwrite=False, test=False, n_max=np.inf):
"""
TODO: duplicate the CAGE/processing code here, so that someone
can run from command line:
convenience function for calling the main daq_to_raw function.
right now, this processes runs sequentially.
"""
from pygama.io.daq_to_raw import daq_to_raw
for run in self.runs:
daq_file = self.paths[run]["daq_path"]
raw_file = self.paths[run]["raw_path"]
if raw_file is not None and overwrite is False:
print("file exists, overwrite flag isn't set. continuing ...")
continue
if test:
print("test mode (dry run), processing Tier 0 file:\n ", daq_file)
continue
daq_to_raw(daq_file, run, suffix=self.config["suffix"],
verbose=test, output_dir=self.raw_dir,
overwrite=overwrite, n_max=n_max, config=self.config)
