def main():
r1_paths = dict(
off=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13268_r1.tio"),
on_50=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13272_r1.tio"),
on_3=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13267_r1.tio")
)
output = get_data("d190520_charge_extraction/data/charge.h5")
poi = 2004
reader = ReaderR1(list(r1_paths.values())[0])
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
extractors = dict(
cc_nn=(CrossCorrelationNeighbour(**kw), 'charge_cc_nn'),
)
for width in range(1, 15):
extractors[f'sliding_{width}'] = (
SlidingWindowNeighbour(**kw, window_size=width),
"charge_sliding_nn"
)
for shift in range(-3, 8):
extractors[f'peak_{width}_{shift}'] = (
CtapipeNeighbourPeakIntegrator(
**kw, window_size=width, window_shift=shift
), "charge_nn"
)
with HDF5Writer(output) as writer:
for key, path in r1_paths.items():
reader = ReaderR1(path, max_events=500)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if reader.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
global_params = dict(
key=key,
iev=iev,
pixel=poi,
)
for name, (extractor, column) in extractors.items():
params = global_params.copy()
params['extractor'] = name
params['charge'] = extractor.process(wfs)[column][poi]
df = pd.DataFrame(params, index=[0])
writer.append(df, key='data')
def main():
description = 'Extract the reference pulse shape from lab measurements'
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
action='store',
required=True,
help='path to the TIO r1 run file')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
args = parser.parse_args()
source = ReaderR1(args.input_path, args.max_events)
t_shift = 60
ref_pulse = get_average_wf(source, t_shift)
ref_pulse /= np.trapz(ref_pulse)
x_ref_pulse = np.arange(ref_pulse.size) * 1E-9
p_wf = WaveformPlotter("Reference Pulse", "mV", "s")
p_wf.add(x_ref_pulse, ref_pulse)
p_wf.save("checs_reference_pulse.pdf")
ref_save = np.column_stack((x_ref_pulse, ref_pulse))
np.savetxt("checs_reference_pulse.txt", ref_save, fmt='%.5e')
def test_waveform_copy():
"""
Test that the waveforms are not altered by any WaveformReducer
"""
reader = ReaderR1(get_file("chec_r1.tio"))
kwargs = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
waveforms = reader[0]
test_waveforms = np.copy(waveforms)
all_reducers = child_subclasses(WaveformReducer)
for r in all_reducers:
try:
reducer = r(**kwargs)
reducer.process(waveforms)
if not (waveforms == test_waveforms).all():
raise ValueError("WaveformReducer {} alters the waveforms!"
.format(r.__name__))
except ImportError:
continue
def open_runlist_r1(path, open_readers=True, max_events=None):
from CHECLabPy.core.io import ReaderR1
df = read_runlist(path)
input_dir = os.path.dirname(path)
input_run_path = os.path.join(input_dir, "Run{:05d}_r1.tio")
df['path'] = [input_run_path.format(i) for i in df.index]
if open_readers:
df['reader'] = [ReaderR1(fp, max_events) for fp in df['path'].values]
return df
def test_readerr1():
reader = ReaderR1(get_file("chec_r1.tio"))
n_events = reader.n_events
count = 0
for _ in reader:
count += 1
assert count > 0
assert count == n_events
def main():
description = ('Reduce a *_r1.tio file into a *_dl1.hdf5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the TIO r1 run files')
parser.add_argument('-m',
'--monitor',
dest='monitor',
action='store',
help='path to the monitor file (OPTIONAL)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
parser.add_argument('-p',
'--plot',
dest='plot',
action='store_true',
help="Plot stages for waveform reducers")
args = parser.parse_args()
if args.configuration:
config = json.loads(args.configuration)
config_string = args.configuration
else:
config = {}
config_string = ""
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
reader = ReaderR1(input_path, args.max_events)
n_events = reader.n_events
n_modules = reader.n_modules
n_pixels = reader.n_pixels
n_samples = reader.n_samples
n_cells = reader.n_cells
pixel_array = np.arange(n_pixels)
camera_version = reader.camera_version
mapping = reader.mapping
if 'reference_pulse_path' not in config:
config['reference_pulse_path'] = reader.reference_pulse_path
kwargs = dict(n_pixels=n_pixels,
n_samples=n_samples,
plot=args.plot,
mapping=mapping,
**config)
reducer = WaveformReducerFactory.produce(args.reducer, **kwargs)
baseline_subtractor = BaselineSubtractor(reader)
input_path = reader.path
output_path = args.output_path
if not output_path:
output_path = input_path.rsplit('_r1', 1)[0] + "_dl1.h5"
with DL1Writer(output_path, n_events * n_pixels,
args.monitor) as writer:
t_cpu = 0
start_time = 0
desc = "Processing events"
for waveforms in tqdm(reader, total=n_events, desc=desc):
iev = reader.index
t_tack = reader.current_tack
t_cpu_sec = reader.current_cpu_s
t_cpu_ns = reader.current_cpu_ns
t_cpu = pd.to_datetime(np.int64(t_cpu_sec * 1E9) +
np.int64(t_cpu_ns),
unit='ns')
fci = reader.first_cell_ids
if not start_time:
start_time = t_cpu
waveforms_bs = baseline_subtractor.subtract(waveforms)
bs = baseline_subtractor.baseline
params = reducer.process(waveforms_bs)
df_ev = pd.DataFrame(
dict(iev=iev,
pixel=pixel_array,
first_cell_id=fci,
t_cpu=t_cpu,
t_tack=t_tack,
baseline_subtracted=bs,
**params))
writer.append_event(df_ev)
sn_dict = {}
for tm in range(n_modules):
sn_dict['TM{:02d}_SN'.format(tm)] = reader.get_sn(tm)
metadata = dict(source="CHECLabPy",
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events,
n_modules=n_modules,
n_pixels=n_pixels,
n_samples=n_samples,
n_cells=n_cells,
start_time=start_time,
end_time=t_cpu,
camera_version=camera_version,
reducer=reducer.__class__.__name__,
configuration=config_string,
**sn_dict)
writer.add_metadata(**metadata)
writer.add_mapping(mapping)
def main():
description = ('Obtain the correction factor for a charge extraction '
'approach defined in CHECLabPy with respect to the '
'CrossCorrelation method')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
action='store',
required=True,
help='path to the runlist.txt file for '
'a dynamic range run')
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
args = parser.parse_args()
config = {}
if args.configuration:
config = json.loads(args.configuration)
output_dir = os.path.dirname(args.input_path)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print("Created directory: {}".format(output_dir))
df_runs = open_runlist_r1(args.input_path, open_readers=False)
df_runs = df_runs.loc[(df_runs['pe_expected'] > 50)
& (df_runs['pe_expected'] < 100)]
df_runs['reader'] = [ReaderR1(fp, 100) for fp in df_runs['path'].values]
n_rows = df_runs.index.size
first_reader = df_runs.iloc[0]['reader']
n_pixels = first_reader.n_pixels
n_samples = first_reader.n_samples
mapping = first_reader.mapping
if 'reference_pulse_path' not in config:
config['reference_pulse_path'] = first_reader.reference_pulse_path
pixel_aray = np.arange(n_pixels)
reducer_dict = dict()
reducers = ['CrossCorrelation', args.reducer]
print("Investigating reducers: {}".format(reducers))
print("Reducer Config: {}".format(config))
for reducer_name in reducers:
reducer = WaveformReducerFactory.produce(reducer_name,
n_pixels=n_pixels,
n_samples=n_samples,
extract_charge_only=True,
mapping=mapping,
**config)
reducer_dict[reducer_name] = reducer
df_list = []
desc0 = "Looping over runs"
it = enumerate(df_runs.iterrows())
for i, (_, row) in tqdm(it, total=n_rows, desc=desc0):
reader = row['reader']
attenuation = row['fw_atten']
baseline_subtractor = BaselineSubtractor(reader)
n_events = reader.n_events
desc1 = "Processing events"
a = np.zeros((2048, n_samples))
for waveforms in tqdm(reader, total=n_events, desc=desc1):
iev = reader.index
waveforms_bs = baseline_subtractor.subtract(waveforms)
for reducer_name, reducer in reducer_dict.items():
charge = reducer.process(waveforms_bs)['charge']
df_list.append(
pd.DataFrame(
dict(
transmission=1 / attenuation,
iev=iev,
pixel=pixel_aray,
reducer=reducer_name,
charge=charge,
)))
df = pd.concat(df_list)
poi = 1920
df_p = df.loc[df['pixel'] == poi]
m_dict = {}
p_scatter = Scatter("Transmission", "Charge (mV)")
for reducer_name in reducers:
df_r = df_p.loc[df_p['reducer'] == reducer_name]
f = dict(charge=['mean', 'std'])
df_agg = df_r.groupby('transmission').agg(f)
x = df_agg.index.values
y = df_agg['charge']['mean'].values
yerr = df_agg['charge']['std'].values
xp = df_r['transmission'].values
yp = df_r['charge'].values
c, m = polyfit(x, y, np.arange(1, 2))
label = reducer_name + " (m={:.3f}, c(fixed)={:.3f})".format(m, c)
p_scatter.add(x, y, yerr=yerr, m=m, c=c, fmt='o', label=label)
m_dict[reducer_name] = m
p_scatter.add_legend(loc='upper left')
path = os.path.join(output_dir, "charge_corrections.pdf".format(poi))
p_scatter.save(path)
ref_reducer = 'CrossCorrelation'
norm_gradient_dict = {
n: float("{:.3f}".format(m_dict[ref_reducer] / v))
for n, v in m_dict.items()
}
print(norm_gradient_dict)
path = os.path.join(output_dir, "charge_corrections.yml")
with open(path, 'w') as file:
yaml.safe_dump(norm_gradient_dict, file, default_flow_style=False)
def main():
runlist_path = get_astri_2019("d2019-04-23_nudges/bright_50pe/runlist.txt")
df_runlist = pd.read_csv(runlist_path, sep='\t')
output = get_astri_2019("d2019-04-23_nudges/bright_50pe/charge.h5")
with HDF5Writer(output) as writer:
mapping = None
for _, row in df_runlist.iterrows():
run = row['run']
nudge = int(row['nudge'])
path = join(dirname(runlist_path), f"Run{run:05d}_r1.tio")
reader = ReaderR1(path, max_events=500)
mapping = reader.mapping
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
extractor_cc = CrossCorrelation(**kw)
extractor_onsky_calib = OnskyCalibExtractor(**kw)
extractor_onsky = OnskyExtractor(**kw)
common = Common(**kw, _disable_by_default=True, waveform_max=True)
pixel_array = np.arange(reader.n_pixels)
monitor = get_nudge_and_temperature_from_reader(reader)
nudge_from_dac, temperature = monitor
assert nudge == nudge_from_dac
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if wfs.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
cc = extractor_cc.process(wfs)['charge_cc']
onsky_calib = extractor_onsky_calib.process(
wfs)['charge_onskycalib']
onsky = extractor_onsky.process(wfs)['charge_onsky']
waveform_max = common.process(wfs)['waveform_max']
params = dict(
nudge=nudge,
nudge_from_dac=nudge_from_dac,
temperature=temperature,
iev=iev,
pixel=pixel_array,
cc=cc,
onsky_calib=onsky_calib,
onsky=onsky,
waveform_max=waveform_max,
)
df = pd.DataFrame(params)
writer.append(df, key='data')
writer.add_mapping(mapping)
def test_readerr1_with_r0():
with pytest.raises(IOError):
reader = ReaderR1(get_file("targetmodule_r0.tio"))
def test_readerr1_single_module():
reader = ReaderR1(get_file("targetmodule_r1.tio"))
assert reader.n_pixels == 64
assert reader[0].shape[0] == 64
def test_readerr1_getitem():
reader = ReaderR1(get_file("chec_r1.tio"))
event = reader[1]
assert event.shape == (reader.n_pixels, reader.n_samples)
assert reader.index == 1
def obtain_readers(files, max_events):
return [ReaderR1(fp, max_events) for fp in files]