def main():
r1_paths = dict(
off=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13268_r1.tio"),
on_50=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13272_r1.tio"),
on_3=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13267_r1.tio")
)
output = get_data("d190520_charge_extraction/data/charge.h5")
poi = 2004
reader = ReaderR1(list(r1_paths.values())[0])
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
extractors = dict(
cc_nn=(CrossCorrelationNeighbour(**kw), 'charge_cc_nn'),
)
for width in range(1, 15):
extractors[f'sliding_{width}'] = (
SlidingWindowNeighbour(**kw, window_size=width),
"charge_sliding_nn"
)
for shift in range(-3, 8):
extractors[f'peak_{width}_{shift}'] = (
CtapipeNeighbourPeakIntegrator(
**kw, window_size=width, window_shift=shift
), "charge_nn"
)
with HDF5Writer(output) as writer:
for key, path in r1_paths.items():
reader = ReaderR1(path, max_events=500)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if reader.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
global_params = dict(
key=key,
iev=iev,
pixel=poi,
)
for name, (extractor, column) in extractors.items():
params = global_params.copy()
params['extractor'] = name
params['charge'] = extractor.process(wfs)[column][poi]
df = pd.DataFrame(params, index=[0])
writer.append(df, key='data')
def main():
# paths = glob(get_astri_2019("d2019-04-23_nudges/mc/*.simtel.gz"))
# output = get_astri_2019("d2019-04-23_nudges/mc/charge.h5")
paths = glob(get_astri_2019("d2019-04-23_nudges/mc_191011/*.simtel.gz"))
output = get_astri_2019("d2019-04-23_nudges/mc_191011/charge.h5")
with HDF5Writer(output) as writer:
for path in paths:
reader = SimtelReader(path)
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
baseline_subtractor = BaselineSubtractor(reader)
extractor_onsky = OnskyExtractor(**kw)
common = Common(**kw, _disable_by_default=True, waveform_max=True)
pixel_array = np.arange(reader.n_pixels)
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
wfs = baseline_subtractor.subtract(wfs)
params = dict(
iev=iev,
pixel=pixel_array,
onsky=extractor_onsky.process(wfs)['charge_onsky'],
waveform_max=common.process(wfs)['waveform_max'],
mc_true=wfs.mc_true,
)
df = pd.DataFrame(params)
writer.append(df, key='data')
def main():
paths = glob("/Users/Jason/Data/d2019-04-23_nudges/mc/*.simtel.gz")
output = get_data("d190520_charge_extraction/mc/charge.h5")
# Use first run to initialise extractors
reader = SimtelReader(paths[0])
pixel_array = np.arange(reader.n_pixels)
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
extractors = dict(
cc_local=(
CrossCorrelationLocal(**kw), "charge_cc_local"
),
cc_nn=(
CrossCorrelationNeighbour(**kw), "charge_cc_nn"
),
local_4_2=(
CtapipeLocalPeakIntegrator(
**kw, window_size=4, window_shift=2
), "charge_local"
),
local_8_4=(
CtapipeLocalPeakIntegrator(
**kw, window_size=8, window_shift=4
), "charge_local"
),
local_6_3=(
CtapipeLocalPeakIntegrator(
**kw, window_size=6, window_shift=3
), "charge_local"
),
nn_4_2=(
CtapipeNeighbourPeakIntegrator(
**kw, window_size=4, window_shift=2
), "charge_nn"
),
nn_8_4=(
CtapipeNeighbourPeakIntegrator(
**kw, window_size=8, window_shift=4
), "charge_nn"
),
nn_6_3=(
CtapipeNeighbourPeakIntegrator(
**kw, window_size=6, window_shift=3
), "charge_nn"
),
sliding_local_4=(
SlidingWindowLocal(**kw, window_size=4), "charge_sliding_local"
),
sliding_local_8=(
SlidingWindowLocal(**kw, window_size=8), "charge_sliding_local"
),
sliding_local_6=(
SlidingWindowLocal(**kw, window_size=6), "charge_sliding_local"
),
sliding_nn_4=(
SlidingWindowNeighbour(**kw, window_size=4), "charge_sliding_nn"
),
sliding_nn_8=(
SlidingWindowNeighbour(**kw, window_size=8), "charge_sliding_nn"
),
sliding_nn_6=(
SlidingWindowNeighbour(**kw, window_size=6), "charge_sliding_nn"
),
)
with HDF5Writer(output) as writer:
for ipath, path in enumerate(paths):
print(f"Processing file {ipath+1}/{len(paths)}")
reader = SimtelReader(path)
baseline_subtractor = BaselineSubtractor(reader)
n_events = reader.n_events
for waveforms in tqdm(reader, total=n_events):
params = dict()
# TODO: Baseline subtraction
waveforms = baseline_subtractor.subtract(waveforms)
for key, (extractor, column) in extractors.items():
params[key] = extractor.process(waveforms)[column]
params['iobs'] = ipath
params['iev'] = waveforms.iev
params['pixel'] = pixel_array
params['true'] = waveforms.mc_true
writer.append(pd.DataFrame(params), key='data')
def main():
description = ('Reduce a waveform file into a *_dl1.h5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the file containing waveforms '
'(TIO or simtel)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-c',
'--config',
dest='config_path',
help="Path to config file. If no path is given, "
"then the default columns will be stored.")
args = parser.parse_args()
time_calibrator = TimeCalibrator()
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
reader = WaveformReader.from_path(input_path, args.max_events)
n_events = reader.n_events
n_modules = reader.n_modules
n_pixels = reader.n_pixels
n_superpixels_per_module = reader.n_superpixels_per_module
n_samples = reader.n_samples
pixel_array = np.arange(n_pixels)
is_mc = isinstance(reader, SimtelReader)
kwargs = dict(
n_pixels=n_pixels,
n_samples=n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
config_path=args.config_path,
)
chain = WaveformReducerChain(**kwargs)
baseline_subtractor = BaselineSubtractor(reader)
input_path = reader.path
output_path = args.output_path
if not output_path:
output_path = (input_path.replace('_r1',
'_dl1').replace('.tio', '.h5'))
if is_mc:
output_path = input_path.replace('.simtel.gz', '_dl1.h5')
with DL1Writer(output_path) as writer:
t_cpu = 0
start_time = 0
n_events_dl1 = 0
n_events_stale = 0
desc = "Processing events"
for waveforms in tqdm(reader, total=n_events, desc=desc):
iev = reader.index
t_cpu = reader.t_cpu
stale = reader.stale.any()
if stale:
n_events_stale += 1
continue
if not start_time:
start_time = t_cpu
shifted = time_calibrator(waveforms)
waveforms_bs = baseline_subtractor.subtract(shifted)
bs = baseline_subtractor.baseline
params = dict(
iev=iev,
pixel=pixel_array,
t_cpu=t_cpu,
t_tack=reader.current_tack,
first_cell_id=reader.first_cell_ids,
baseline_subtracted=bs,
**chain.process(waveforms_bs),
)
if is_mc:
params['mc_true'] = reader.mc_true
writer.append(pd.DataFrame(params),
key='data',
expectedrows=n_events * n_pixels)
if is_mc:
writer.append(pd.DataFrame([reader.mc]),
key='mc',
expectedrows=n_events)
writer.append(pd.DataFrame([reader.pointing]),
key='pointing',
expectedrows=n_events)
n_events_dl1 += 1
sn_dict = {}
sipm_temp_dict = {}
primary_temp_dict = {}
dac_dict = {}
hvon_dict = {}
for tm in range(n_modules):
tm_str = f'TM{tm:02}'
sn_dict[tm_str] = reader.get_sn(tm)
sipm_temp_dict[tm_str] = reader.get_sipm_temp(tm)
primary_temp_dict[tm_str] = reader.get_primary_temp(tm)
for sp in range(n_superpixels_per_module):
tm_sp_str = f'TM{tm:02}_SP{sp:02}'
dac_dict[tm_sp_str] = reader.get_sp_dac(tm, sp)
hvon_dict[tm_sp_str] = reader.get_sp_hvon(tm, sp)
metadata = dict(
source="CHECLabPy",
run_id=reader.run_id,
is_mc=is_mc,
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events_dl1,
n_modules=n_modules,
n_pixels=n_pixels,
n_superpixels_per_module=n_superpixels_per_module,
n_samples=n_samples,
start_time=start_time,
end_time=t_cpu,
camera_version=reader.camera_version,
n_cells=reader.n_cells,
n_stale=n_events_stale,
)
config = chain.config
config.pop('mapping', None)
writer.add_mapping(reader.mapping)
writer.add_metadata(name='metadata', **metadata)
writer.add_metadata(name='config', **config)
writer.add_metadata(name='sn', **sn_dict)
writer.add_metadata(name='sipm_temp', **sipm_temp_dict)
writer.add_metadata(name='primary_temp', **sipm_temp_dict)
writer.add_metadata(name='dac', **dac_dict)
writer.add_metadata(name='hvon', **hvon_dict)
if is_mc:
writer.add_metadata(key='mc',
name='mcheader',
**reader.mcheader)
def main():
description = ('Reduce a *_r1.tio file into a *_dl1.hdf5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the TIO r1 run files')
parser.add_argument('-m',
'--monitor',
dest='monitor',
action='store',
help='path to the monitor file (OPTIONAL)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
parser.add_argument('-p',
'--plot',
dest='plot',
action='store_true',
help="Plot stages for waveform reducers")
args = parser.parse_args()
if args.configuration:
config = json.loads(args.configuration)
config_string = args.configuration
else:
config = {}
config_string = ""
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
reader = ReaderR1(input_path, args.max_events)
n_events = reader.n_events
n_modules = reader.n_modules
n_pixels = reader.n_pixels
n_samples = reader.n_samples
n_cells = reader.n_cells
pixel_array = np.arange(n_pixels)
camera_version = reader.camera_version
mapping = reader.mapping
if 'reference_pulse_path' not in config:
config['reference_pulse_path'] = reader.reference_pulse_path
kwargs = dict(n_pixels=n_pixels,
n_samples=n_samples,
plot=args.plot,
mapping=mapping,
**config)
reducer = WaveformReducerFactory.produce(args.reducer, **kwargs)
baseline_subtractor = BaselineSubtractor(reader)
input_path = reader.path
output_path = args.output_path
if not output_path:
output_path = input_path.rsplit('_r1', 1)[0] + "_dl1.h5"
with DL1Writer(output_path, n_events * n_pixels,
args.monitor) as writer:
t_cpu = 0
start_time = 0
desc = "Processing events"
for waveforms in tqdm(reader, total=n_events, desc=desc):
iev = reader.index
t_tack = reader.current_tack
t_cpu_sec = reader.current_cpu_s
t_cpu_ns = reader.current_cpu_ns
t_cpu = pd.to_datetime(np.int64(t_cpu_sec * 1E9) +
np.int64(t_cpu_ns),
unit='ns')
fci = reader.first_cell_ids
if not start_time:
start_time = t_cpu
waveforms_bs = baseline_subtractor.subtract(waveforms)
bs = baseline_subtractor.baseline
params = reducer.process(waveforms_bs)
df_ev = pd.DataFrame(
dict(iev=iev,
pixel=pixel_array,
first_cell_id=fci,
t_cpu=t_cpu,
t_tack=t_tack,
baseline_subtracted=bs,
**params))
writer.append_event(df_ev)
sn_dict = {}
for tm in range(n_modules):
sn_dict['TM{:02d}_SN'.format(tm)] = reader.get_sn(tm)
metadata = dict(source="CHECLabPy",
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events,
n_modules=n_modules,
n_pixels=n_pixels,
n_samples=n_samples,
n_cells=n_cells,
start_time=start_time,
end_time=t_cpu,
camera_version=camera_version,
reducer=reducer.__class__.__name__,
configuration=config_string,
**sn_dict)
writer.add_metadata(**metadata)
writer.add_mapping(mapping)
class DL1Extractor:
def __init__(self, reader, config_path):
self.source = "CHECLabPy"
self.reader = reader
self.pixel_array = np.arange(self.reader.n_pixels)
self.time_calibrator = None
if not self.reader.is_mc:
self.time_calibrator = TimeCalibrator()
self.reducer_kwargs = dict(
n_pixels=self.reader.n_pixels,
n_samples=self.reader.n_samples,
mapping=self.reader.mapping,
reference_pulse_path=self.reader.reference_pulse_path,
config_path=config_path,
)
self.chain = WaveformReducerChain(**self.reducer_kwargs)
self.baseline_subtractor = BaselineSubtractor(self.reader)
# Module metadata dicts
self.config = self.chain.config.copy()
self.config.pop('mapping', None)
self.sn = {}
self.sipm_temp = {}
self.primary_temp = {}
self.dac = {}
self.hvon = {}
for tm in range(self.reader.n_modules):
tm_str = f'TM{tm:02}'
self.sn[tm_str] = self.reader.get_sn(tm)
self.sipm_temp[tm_str] = self.reader.get_sipm_temp(tm)
self.primary_temp[tm_str] = self.reader.get_primary_temp(tm)
for sp in range(self.reader.n_superpixels_per_module):
tm_sp_str = f'TM{tm:02}_SP{sp:02}'
self.dac[tm_sp_str] = self.reader.get_sp_dac(tm, sp)
self.hvon[tm_sp_str] = self.reader.get_sp_hvon(tm, sp)
@staticmethod
def skip_event(waveforms):
return waveforms.stale or waveforms.missing_packets
@property
def metadata(self):
return dict(
source=self.source,
run_id=self.reader.run_id,
is_mc=self.reader.is_mc,
date_generated=pd.datetime.now(),
input_path=self.reader.path,
n_modules=self.reader.n_modules,
n_pixels=self.reader.n_pixels,
n_superpixels_per_module=self.reader.n_superpixels_per_module,
n_samples=self.reader.n_samples,
camera_version=self.reader.camera_version,
n_cells=self.reader.n_cells,
)
def __call__(self, waveforms):
# Calibrate timing offsets between pixels
waveforms_tc = waveforms
if self.time_calibrator is not None:
waveforms_tc = self.time_calibrator(waveforms)
# Subtract waveform baselines per pixel
waveforms_bs = self.baseline_subtractor.subtract(waveforms_tc)
bs = self.baseline_subtractor.baseline
params = dict(
iev=waveforms.iev,
pixel=self.pixel_array,
t_cpu=waveforms.t_cpu,
t_tack=waveforms.t_tack,
first_cell_id=waveforms.first_cell_id,
baseline_subtracted=bs,
**self.chain.process(waveforms_bs),
)
if waveforms.mc_true is not None:
params['mc_true'] = waveforms.mc_true
return params
def main():
description = ('Obtain the correction factor for a charge extraction '
'approach defined in CHECLabPy with respect to the '
'CrossCorrelation method')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
action='store',
required=True,
help='path to the runlist.txt file for '
'a dynamic range run')
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
args = parser.parse_args()
config = {}
if args.configuration:
config = json.loads(args.configuration)
output_dir = os.path.dirname(args.input_path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print("Created directory: {}".format(output_dir))
df_runs = open_runlist_r1(args.input_path, open_readers=False)
df_runs = df_runs.loc[(df_runs['pe_expected'] > 50)
& (df_runs['pe_expected'] < 100)]
df_runs['reader'] = [ReaderR1(fp, 100) for fp in df_runs['path'].values]
n_rows = df_runs.index.size
first_reader = df_runs.iloc[0]['reader']
n_pixels = first_reader.n_pixels
n_samples = first_reader.n_samples
mapping = first_reader.mapping
if 'reference_pulse_path' not in config:
config['reference_pulse_path'] = first_reader.reference_pulse_path
pixel_aray = np.arange(n_pixels)
reducer_dict = dict()
reducers = ['CrossCorrelation', args.reducer]
print("Investigating reducers: {}".format(reducers))
print("Reducer Config: {}".format(config))
for reducer_name in reducers:
reducer = WaveformReducerFactory.produce(reducer_name,
n_pixels=n_pixels,
n_samples=n_samples,
extract_charge_only=True,
mapping=mapping,
**config)
reducer_dict[reducer_name] = reducer
df_list = []
desc0 = "Looping over runs"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_rows, desc=desc0):
reader = row['reader']
attenuation = row['fw_atten']
baseline_subtractor = BaselineSubtractor(reader)
n_events = reader.n_events
desc1 = "Processing events"
a = np.zeros((2048, n_samples))
for waveforms in tqdm(reader, total=n_events, desc=desc1):
iev = reader.index
waveforms_bs = baseline_subtractor.subtract(waveforms)
for reducer_name, reducer in reducer_dict.items():
charge = reducer.process(waveforms_bs)['charge']
df_list.append(
pd.DataFrame(
dict(
transmission=1 / attenuation,
iev=iev,
pixel=pixel_aray,
reducer=reducer_name,
charge=charge,
)))
df = pd.concat(df_list)
poi = 1920
df_p = df.loc[df['pixel'] == poi]
m_dict = {}
p_scatter = Scatter("Transmission", "Charge (mV)")
for reducer_name in reducers:
df_r = df_p.loc[df_p['reducer'] == reducer_name]
f = dict(charge=['mean', 'std'])
df_agg = df_r.groupby('transmission').agg(f)
x = df_agg.index.values
y = df_agg['charge']['mean'].values
yerr = df_agg['charge']['std'].values
xp = df_r['transmission'].values
yp = df_r['charge'].values
c, m = polyfit(x, y, np.arange(1, 2))
label = reducer_name + " (m={:.3f}, c(fixed)={:.3f})".format(m, c)
p_scatter.add(x, y, yerr=yerr, m=m, c=c, fmt='o', label=label)
m_dict[reducer_name] = m
p_scatter.add_legend(loc='upper left')
path = os.path.join(output_dir, "charge_corrections.pdf".format(poi))
p_scatter.save(path)
ref_reducer = 'CrossCorrelation'
norm_gradient_dict = {
n: float("{:.3f}".format(m_dict[ref_reducer] / v))
for n, v in m_dict.items()
}
print(norm_gradient_dict)
path = os.path.join(output_dir, "charge_corrections.yml")
with open(path, 'w') as file:
yaml.safe_dump(norm_gradient_dict, file, default_flow_style=False)
def main():
runlist_path = get_astri_2019("d2019-04-23_nudges/bright_50pe/runlist.txt")
df_runlist = pd.read_csv(runlist_path, sep='\t')
output = get_astri_2019("d2019-04-23_nudges/bright_50pe/charge.h5")
with HDF5Writer(output) as writer:
mapping = None
for _, row in df_runlist.iterrows():
run = row['run']
nudge = int(row['nudge'])
path = join(dirname(runlist_path), f"Run{run:05d}_r1.tio")
reader = ReaderR1(path, max_events=500)
mapping = reader.mapping
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
extractor_cc = CrossCorrelation(**kw)
extractor_onsky_calib = OnskyCalibExtractor(**kw)
extractor_onsky = OnskyExtractor(**kw)
common = Common(**kw, _disable_by_default=True, waveform_max=True)
pixel_array = np.arange(reader.n_pixels)
monitor = get_nudge_and_temperature_from_reader(reader)
nudge_from_dac, temperature = monitor
assert nudge == nudge_from_dac
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if wfs.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
cc = extractor_cc.process(wfs)['charge_cc']
onsky_calib = extractor_onsky_calib.process(
wfs)['charge_onskycalib']
onsky = extractor_onsky.process(wfs)['charge_onsky']
waveform_max = common.process(wfs)['waveform_max']
params = dict(
nudge=nudge,
nudge_from_dac=nudge_from_dac,
temperature=temperature,
iev=iev,
pixel=pixel_array,
cc=cc,
onsky_calib=onsky_calib,
onsky=onsky,
waveform_max=waveform_max,
)
df = pd.DataFrame(params)
writer.append(df, key='data')
writer.add_mapping(mapping)