def r2d(dg, overwrite=False, nwfs=None, vrb=False): """ """ # print(dg.file_keys) # print(dg.file_keys.columns) with open(f'config_dsp.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) for i, row in dg.file_keys.iterrows(): f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" f_dsp = f"{dg.lh5_dir}/{row['dsp_path']}/{row['dsp_file']}" if "sysn" in f_raw: tmp = {'sysn': 'geds'} # hack for lpgta f_raw = f_raw.format_map(tmp) f_dsp = f_dsp.format_map(tmp) if not overwrite and os.path.exists(f_dsp): print('file exists, overwrite not set, skipping f_dsp:\n ', f_dsp) continue raw_to_dsp(f_raw, f_dsp, dsp_config, n_max=nwfs, verbose=vrb, overwrite=overwrite)
def r2d(dg, overwrite=False, nwfs=None, vrb=False): """ """ # print(dg.fileDB) # print(dg.fileDB.columns) with open(f'{dg.experiment}_dsp.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) for i, row in dg.fileDB.iterrows(): f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" f_dsp = f"{dg.lh5_dir}/{row['dsp_path']}/{row['dsp_file']}" if "sysn" in f_raw: tmp = {'sysn': 'geds'} # hack for lpgta f_raw = f_raw.format_map(tmp) f_dsp = f_dsp.format_map(tmp) raw_to_dsp(f_raw, f_dsp, dsp_config, n_max=nwfs, verbose=vrb, overwrite=overwrite)
def r2d(dg, overwrite=False, nwfs=None, verbose=False, user=False): """ $ ./processing.py -q 'run==[something]' --r2d """ # load default DSP config file dsp_dir = os.path.expandvars('$CAGE_SW/processing/metadata') with open(dsp_dir + '/config_dsp.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) for i, row in dg.fileDB.iterrows(): lh5_dir = dg.lh5_user_dir if user else dg.lh5_dir f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" f_dsp = f"{lh5_dir}/{row['dsp_path']}/{row['dsp_file']}" if "sysn" in f_raw: tmp = {'sysn': 'geds'} # hack for lpgta f_raw = f_raw.format_map(tmp) f_dsp = f_dsp.format_map(tmp) if not overwrite and os.path.exists(f_dsp): print('file exists, overwrite not set, skipping f_dsp:\n ', f_dsp) continue cyc = row['cycle'] if row.skip: print(f'Cycle {cyc} has been marked junk, will not process.') continue # load updated dsp config file (optional) if row.dsp_id > 0: with open(dsp_dir + f'/dsp/dsp_{row.dsp_id:02d}.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) print(f'Using DSP config: {dsp_dir}' + f'/dsp/dsp_{row.dsp_id:02d}.json') # NOTE: there is currently no smart DSP DB lookup here, # so the "db defaults" values in each of the JSON files will be used. print(f'Processing cycle {cyc}') raw_to_dsp(f_raw, f_dsp, dsp_config, n_max=nwfs, verbose=verbose, overwrite=overwrite)
def r2d_file(f_raw, f_dsp, overwrite=True, nwfs=None, vrb=False): """ single-file mode, for testing """ print('raw_to_dsp, single-file mode.') print(' input:', f_raw) print(' output:', f_dsp) # always overwrite if os.path.exists(f_dsp): os.remove(f_dsp) with open('oppi_dsp.json') as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) raw_to_dsp(f_raw, f_dsp, dsp_config, n_max=nwfs, verbose=vrb, overwrite=overwrite)
def r2d(dg, overwrite=False, nwfs=None, verbose=False, user=False): """ $ ./processing.py -q 'run==[something]' --r2d """ # print(dg.fileDB) # print(dg.fileDB.columns) with open( os.path.expandvars( '$CAGE_SW/processing/metadata/config_dsp.json')) as f: dsp_config = json.load(f, object_pairs_hook=OrderedDict) for i, row in dg.fileDB.iterrows(): lh5_dir = dg.lh5_user_dir if user else dg.lh5_dir f_raw = f"{dg.lh5_dir}/{row['raw_path']}/{row['raw_file']}" f_dsp = f"{lh5_dir}/{row['dsp_path']}/{row['dsp_file']}" if "sysn" in f_raw: tmp = {'sysn': 'geds'} # hack for lpgta f_raw = f_raw.format_map(tmp) f_dsp = f_dsp.format_map(tmp) if not overwrite and os.path.exists(f_dsp): print('file exists, overwrite not set, skipping f_dsp:\n ', f_dsp) continue cyc = row['cycle'] if row.skip: print(f'Cycle {cyc} has been marked junk, will not process.') continue print(f'Processing cycle {cyc}') raw_to_dsp(f_raw, f_dsp, dsp_config, n_max=nwfs, verbose=verbose, overwrite=overwrite)
import argparse, os, pathlib import pygama from pygama.io.raw_to_dsp import raw_to_dsp import json from collections import OrderedDict argparser = argparse.ArgumentParser() argparser.add_argument("--metadata", help="metadata path", type=str, required=True) argparser.add_argument("input", help="input file", type=str) argparser.add_argument("output", help="output file", type=str) args = argparser.parse_args() f_config = os.path.join(f"{args.metadata}", "config_dsp.json") with open(f_config) as f: config_dic = json.load(f, object_pairs_hook=OrderedDict) pathlib.Path(os.path.dirname(args.output)).mkdir(parents=True, exist_ok=True) # ToDo: Atomic file creation raw_to_dsp(args.input, args.output, config_dic, verbose=True, overwrite=False)