def main():
r1_paths = dict(
off=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13268_r1.tio"),
on_50=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13272_r1.tio"),
on_3=get_astri_2019("d2019-05-08_ledflashers_dynrange/Run13267_r1.tio")
)
output = get_data("d190520_charge_extraction/data/charge.h5")
poi = 2004
reader = ReaderR1(list(r1_paths.values())[0])
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
extractors = dict(
cc_nn=(CrossCorrelationNeighbour(**kw), 'charge_cc_nn'),
)
for width in range(1, 15):
extractors[f'sliding_{width}'] = (
SlidingWindowNeighbour(**kw, window_size=width),
"charge_sliding_nn"
)
for shift in range(-3, 8):
extractors[f'peak_{width}_{shift}'] = (
CtapipeNeighbourPeakIntegrator(
**kw, window_size=width, window_shift=shift
), "charge_nn"
)
with HDF5Writer(output) as writer:
for key, path in r1_paths.items():
reader = ReaderR1(path, max_events=500)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if reader.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
global_params = dict(
key=key,
iev=iev,
pixel=poi,
)
for name, (extractor, column) in extractors.items():
params = global_params.copy()
params['extractor'] = name
params['charge'] = extractor.process(wfs)[column][poi]
df = pd.DataFrame(params, index=[0])
writer.append(df, key='data')
def __init__(self, reader, config_path):
self.source = "CHECLabPy"
self.reader = reader
self.pixel_array = np.arange(self.reader.n_pixels)
self.time_calibrator = None
if not self.reader.is_mc:
self.time_calibrator = TimeCalibrator()
self.reducer_kwargs = dict(
n_pixels=self.reader.n_pixels,
n_samples=self.reader.n_samples,
mapping=self.reader.mapping,
reference_pulse_path=self.reader.reference_pulse_path,
config_path=config_path,
)
self.chain = WaveformReducerChain(**self.reducer_kwargs)
self.baseline_subtractor = BaselineSubtractor(self.reader)
# Module metadata dicts
self.config = self.chain.config.copy()
self.config.pop('mapping', None)
self.sn = {}
self.sipm_temp = {}
self.primary_temp = {}
self.dac = {}
self.hvon = {}
for tm in range(self.reader.n_modules):
tm_str = f'TM{tm:02}'
self.sn[tm_str] = self.reader.get_sn(tm)
self.sipm_temp[tm_str] = self.reader.get_sipm_temp(tm)
self.primary_temp[tm_str] = self.reader.get_primary_temp(tm)
for sp in range(self.reader.n_superpixels_per_module):
tm_sp_str = f'TM{tm:02}_SP{sp:02}'
self.dac[tm_sp_str] = self.reader.get_sp_dac(tm, sp)
self.hvon[tm_sp_str] = self.reader.get_sp_hvon(tm, sp)
def main():
input_path = "event_list.txt"
output_path = "cherenkov.npz"
df = pd.read_csv(input_path, sep='\t')
first_path = df.iloc[0]['path'].replace("_hillas.h5", "_r1.tio")
first_reader = TIOReader(first_path)
n_pixels = first_reader.n_pixels
n_samples = first_reader.n_samples
mapping = first_reader.mapping
mapping.metadata['size'] *= 1.01 # TODO: WHY?!
reference_pulse_path = first_reader.reference_pulse_path
geom = get_ctapipe_camera_geometry(mapping)
charge_extractor = OnskyExtractor(
n_pixels,
n_samples,
mapping=mapping,
reference_pulse_path=reference_pulse_path,
)
time_calibrator = TimeCalibrator()
# Open all files
hillas_paths = set()
for _, row in df.iterrows():
hillas_paths.add(row['path'])
readers = dict()
amplitude_calibrators = dict()
for path in hillas_paths:
r1_path = path.replace("_hillas.h5", "_r1.tio")
reader = TIOReader(r1_path)
nudge, temperature = get_nudge_and_temperature_from_reader(reader)
amplitude_calibrator = OnskyAmplitudeCalibrator(nudge, temperature)
readers[path] = reader
amplitude_calibrators[path] = amplitude_calibrator
n_events = df.index.size
frames_array = []
min_array = []
max_array = []
desc = "Looping over events"
for i, row in tqdm(df.iterrows(), total=n_events, desc=desc):
if i >= n_events:
break
hillas_path = row['path']
iev = row['iev']
iobs = row['iobs']
reader = readers[hillas_path]
amplitude_calibrator = amplitude_calibrators[hillas_path]
waveforms = reader[iev]
shifted = time_calibrator(waveforms)
extracted = charge_extractor.process(shifted)
charge = extracted['charge_onsky']
time = extracted['t_onsky']
photons = amplitude_calibrator(charge, np.arange(n_pixels))
pe = photons * 0.25
mask = obtain_cleaning_mask(geom, photons, time)
if not mask.any():
msg = f"No pixels survived cleaning for: RUN {iobs} IEV {iev}"
raise ValueError(msg)
photons_ma = np.ma.masked_array(photons, mask=~mask)
min_pixel = photons_ma.argmin()
max_pixel = photons_ma.argmax()
min_image = -4
max_image = 0.7 * pe.max()
min_gf = shifted[max_pixel, :20].min()
max_gf = shifted[max_pixel].max() * 0.8
st = int(np.min(time[mask]) - 3)
et = int(np.max(time[mask]) + 6)
st = st if st > 0 else 0
et = et if et < n_samples else n_samples
frames = shifted[:, st:et:4]
min_ = np.full(frames.shape[-1], min_gf)
max_ = np.full(frames.shape[-1], max_gf)
frames_array.append(frames)
min_array.append(min_)
max_array.append(max_)
np.savez(
output_path,
frames=np.column_stack(frames_array),
min=np.concatenate(min_array),
max=np.concatenate(max_array),
)
def main():
description = ('Reduce a waveform file into a *_dl1.h5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the file containing waveforms '
'(TIO or simtel)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-c',
'--config',
dest='config_path',
help="Path to config file. If no path is given, "
"then the default columns will be stored.")
args = parser.parse_args()
time_calibrator = TimeCalibrator()
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
reader = WaveformReader.from_path(input_path, args.max_events)
n_events = reader.n_events
n_modules = reader.n_modules
n_pixels = reader.n_pixels
n_superpixels_per_module = reader.n_superpixels_per_module
n_samples = reader.n_samples
pixel_array = np.arange(n_pixels)
is_mc = isinstance(reader, SimtelReader)
kwargs = dict(
n_pixels=n_pixels,
n_samples=n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
config_path=args.config_path,
)
chain = WaveformReducerChain(**kwargs)
baseline_subtractor = BaselineSubtractor(reader)
input_path = reader.path
output_path = args.output_path
if not output_path:
output_path = (input_path.replace('_r1',
'_dl1').replace('.tio', '.h5'))
if is_mc:
output_path = input_path.replace('.simtel.gz', '_dl1.h5')
with DL1Writer(output_path) as writer:
t_cpu = 0
start_time = 0
n_events_dl1 = 0
n_events_stale = 0
desc = "Processing events"
for waveforms in tqdm(reader, total=n_events, desc=desc):
iev = reader.index
t_cpu = reader.t_cpu
stale = reader.stale.any()
if stale:
n_events_stale += 1
continue
if not start_time:
start_time = t_cpu
shifted = time_calibrator(waveforms)
waveforms_bs = baseline_subtractor.subtract(shifted)
bs = baseline_subtractor.baseline
params = dict(
iev=iev,
pixel=pixel_array,
t_cpu=t_cpu,
t_tack=reader.current_tack,
first_cell_id=reader.first_cell_ids,
baseline_subtracted=bs,
**chain.process(waveforms_bs),
)
if is_mc:
params['mc_true'] = reader.mc_true
writer.append(pd.DataFrame(params),
key='data',
expectedrows=n_events * n_pixels)
if is_mc:
writer.append(pd.DataFrame([reader.mc]),
key='mc',
expectedrows=n_events)
writer.append(pd.DataFrame([reader.pointing]),
key='pointing',
expectedrows=n_events)
n_events_dl1 += 1
sn_dict = {}
sipm_temp_dict = {}
primary_temp_dict = {}
dac_dict = {}
hvon_dict = {}
for tm in range(n_modules):
tm_str = f'TM{tm:02}'
sn_dict[tm_str] = reader.get_sn(tm)
sipm_temp_dict[tm_str] = reader.get_sipm_temp(tm)
primary_temp_dict[tm_str] = reader.get_primary_temp(tm)
for sp in range(n_superpixels_per_module):
tm_sp_str = f'TM{tm:02}_SP{sp:02}'
dac_dict[tm_sp_str] = reader.get_sp_dac(tm, sp)
hvon_dict[tm_sp_str] = reader.get_sp_hvon(tm, sp)
metadata = dict(
source="CHECLabPy",
run_id=reader.run_id,
is_mc=is_mc,
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events_dl1,
n_modules=n_modules,
n_pixels=n_pixels,
n_superpixels_per_module=n_superpixels_per_module,
n_samples=n_samples,
start_time=start_time,
end_time=t_cpu,
camera_version=reader.camera_version,
n_cells=reader.n_cells,
n_stale=n_events_stale,
)
config = chain.config
config.pop('mapping', None)
writer.add_mapping(reader.mapping)
writer.add_metadata(name='metadata', **metadata)
writer.add_metadata(name='config', **config)
writer.add_metadata(name='sn', **sn_dict)
writer.add_metadata(name='sipm_temp', **sipm_temp_dict)
writer.add_metadata(name='primary_temp', **sipm_temp_dict)
writer.add_metadata(name='dac', **dac_dict)
writer.add_metadata(name='hvon', **hvon_dict)
if is_mc:
writer.add_metadata(key='mc',
name='mcheader',
**reader.mcheader)
def main():
path = get_data("d190717_alpha/wobble.h5")
with pd.HDFStore(path, mode='r') as store:
df = store['data'].loc[::4]
mapping = store['mapping']
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning)
mapping.metadata = store.get_storer('mapping').attrs.metadata
tc = TimeCalibrator()
geom = get_ctapipe_camera_geometry(mapping)
n_row = df.index.size
p_camera = CameraMovie(mapping, get_plot(
"d190717_alpha/wobble_animation_goldfish/frames/{:04d}.png"
))
for _, row in tqdm(df.iterrows(), total=n_row):
timestamp = row['timestamp']
iobs = row['iobs']
iev = row['iev']
x_src = row['x_src']
y_src = row['y_src']
dl1 = row['dl1'].values
time = row['dl1_pulse_time'].values
r1 = row['r1']
x_cog = row['x_cog']
y_cog = row['y_cog']
psi = row['psi']
p_camera.set_source_position(x_src, y_src)
n_pixels, n_samples = r1.shape
shifted = tc(r1)
mask = obtain_cleaning_mask(geom, dl1, time)
if not mask.any():
msg = f"No pixels survived cleaning for: RUN {iobs} IEV {iev}"
print(msg)
continue
# raise ValueError(msg)
dl1_ma = np.ma.masked_array(dl1, mask=~mask)
min_pixel = dl1_ma.argmin()
max_pixel = dl1_ma.argmax()
min_image = -4
max_image = 0.7 * dl1.max()
min_gf = shifted[max_pixel, :20].min()
max_gf = shifted[max_pixel].max() * 0.8
st = int(np.min(time[mask]) - 3)
et = int(np.max(time[mask]) + 6)
st = st if st > 0 else 0
et = et if et < n_samples else n_samples
# embed()
p_camera.set_image(dl1, min_image, max_image)
for t in range(st, et, 3):
slice_ = shifted[:, t]
p_camera.set_timestamp(timestamp + pd.Timedelta(f"{t}ns"))
p_camera.set_goldfish(slice_, min_gf, max_gf)
p_camera.save_frame()
def main():
path = "/Users/Jason/Data/d2019-04-23_nudges/bright_50pe/Run09095_r1.tio"
reader = TIOReader(path)
n_events = reader.n_events
n_pixels = reader.n_pixels
n_samples = reader.n_samples
pixel_array = np.arange(n_pixels)
time_calibrator = TimeCalibrator()
extractor = Timing(n_pixels, n_samples)
df_list = []
for wfs in tqdm(reader, total=n_events):
iev = wfs.iev
shifted = time_calibrator(wfs)
params = extractor.process(wfs)
params_shifted = extractor.process(shifted)
df_list.append(
pd.DataFrame(
dict(
iev=iev,
pixel=pixel_array,
t_pulse=params['t_pulse'],
t_pulse_corrected=params_shifted['t_pulse'],
)))
df = pd.concat(df_list, ignore_index=True)
pm = PixelMasks()
dead = np.where(pm.all_mask)
mask = ~np.isin(df['pixel'].values, dead)
df = df.iloc[mask]
df_ev = df.groupby('iev').mean()
mean_timing = np.repeat(df_ev['t_pulse'], n_pixels - pm.all_mask.sum())
mean_timing_c = np.repeat(df_ev['t_pulse'], n_pixels - pm.all_mask.sum())
df['t_pulse'] -= mean_timing.values
df['t_pulse_corrected'] -= mean_timing_c.values
df_pix = df.groupby('pixel').mean()
pixel = df_pix.index.values
t_pulse = df_pix['t_pulse'].values
t_pulse_corrected = df_pix['t_pulse_corrected'].values
image = np.full(n_pixels, np.nan)
image[pixel] = t_pulse
mean = np.nanmean(image)
std = np.nanstd(image)
ci = CameraImage.from_mapping(reader.mapping)
ci.image = image
ci.add_colorbar()
ci.ax.set_title(f"MEAN = {mean:.2f}, STDDEV = {std:.2f}")
ci.save(join(DIR, "before.pdf"))
image = np.full(n_pixels, np.nan)
image[pixel] = t_pulse_corrected
mean = np.nanmean(image)
std = np.nanstd(image)
ci = CameraImage.from_mapping(reader.mapping)
ci.image = image
ci.add_colorbar()
ci.ax.set_title(f"MEAN = {mean:.2f}, STDDEV = {std:.2f}")
ci.save(join(DIR, "after.pdf"))
def process(dataset):
name = dataset.__class__.__name__
directory = dataset.directory
event_dict = dataset.events
output_dir = get_plot(f"d190506_astri_publicity/{name}")
goldfish_path = join(output_dir, 'goldfish.gif')
image_path = join(output_dir, 'image.gif')
first_run = list(event_dict.keys())[0]
first_reader = TIOReader(join(directory, first_run + "_r1.tio"))
mapping = first_reader.mapping
p_image = CameraAnimation.from_mapping(mapping)
p_image.add_colorbar("Amplitude (Photoelectons)")
p_goldfish = CameraAnimation.from_mapping(mapping)
p_goldfish.add_colorbar("Amplitude (mV)")
for irun, run in enumerate(event_dict.keys()):
print(f"Processing run: {run} ({irun+1}/{len(event_dict)})")
path = join(directory, run + "_r1.tio")
reader = TIOReader(path)
n_events = reader.n_events
n_pixels = reader.n_pixels
n_samples = reader.n_samples
reference_pulse_path = reader.reference_pulse_path
geom = get_ctapipe_camera_geometry(mapping, plate_scale=37.56e-3)
charge_extractor = CrossCorrelationNeighbour(
n_pixels,
n_samples,
mapping=mapping,
reference_pulse_path=reference_pulse_path,
)
time_calibrator = TimeCalibrator()
nudge, temperature = get_nudge_and_temperature_from_reader(reader)
amplitude_calibrator = AstriAmplitudeCalibrator(nudge, temperature)
for event in tqdm(event_dict[run]):
wfs = reader[event]
shifted = time_calibrator(wfs)
extracted = charge_extractor.process(shifted)
charge = extracted['charge_cc_nn']
time = extracted['t_cc_nn']
photons = amplitude_calibrator(charge, np.arange(n_pixels))
pe = photons * 0.25
mask = tailcut(geom, photons, time)
if not mask.any():
msg = f"No pixels survived cleaning for: {run} {event}"
# raise ValueError(msg)
print(msg)
three_largest = np.argsort(photons)[:-3]
mask = np.zeros(n_pixels, dtype=np.bool)
mask[three_largest] = True
photons_nan = photons
photons_nan[~mask] = np.nan
min_pixel = np.nanargmin(photons_nan)
max_pixel = np.nanargmax(photons_nan)
min_image = 0
max_image = 0.7 * pe.max()
min_goldfish = shifted[max_pixel, :20].min()
max_goldfish = shifted[max_pixel].max() * 0.8
p_image.add_image(run, event, 0, pe, min_image, max_image)
start_time = int(np.min(time[mask]) - 2)
end_time = int(np.max(time[mask]) + 2)
for t in range(start_time, end_time):
if 0 <= t < n_samples:
p_goldfish.add_image(run, event, t, shifted[:, t],
min_goldfish, max_goldfish)
p_image.animate(300, image_path)
p_goldfish.animate(20, goldfish_path)
def main():
description = 'Loop over R0 or R1 file and plot camera images'
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--file',
dest='input_path',
required=True,
help='path to a hillas list file created by '
'generate_list_from_hillas')
parser.add_argument('-n',
'--max_events',
dest='max_events',
type=int,
help='number of events to plot')
args = parser.parse_args()
input_path = args.input_path
output_dir = splitext(input_path)[0]
max_events = args.max_events
df = pd.read_csv(input_path, sep='\t')
first_path = df.iloc[0]['path'].replace("_hillas.h5", "_r1.tio")
first_reader = TIOReader(first_path)
n_pixels = first_reader.n_pixels
n_samples = first_reader.n_samples
mapping = first_reader.mapping
mapping.metadata['size'] *= 1.01 # TODO: WHY?!
reference_pulse_path = first_reader.reference_pulse_path
geom = get_ctapipe_camera_geometry(mapping, plate_scale=37.56e-3)
charge_extractor = OnskyExtractor(
n_pixels,
n_samples,
mapping=mapping,
reference_pulse_path=reference_pulse_path,
)
time_calibrator = TimeCalibrator()
# Open all files
hillas_paths = set()
for _, row in df.iterrows():
hillas_paths.add(row['path'])
readers = dict()
amplitude_calibrators = dict()
for path in hillas_paths:
r1_path = path.replace("_hillas.h5", "_r1.tio")
reader = TIOReader(r1_path)
nudge, temperature = get_nudge_and_temperature_from_reader(reader)
amplitude_calibrator = OnskyAmplitudeCalibrator(nudge, temperature)
readers[path] = reader
amplitude_calibrators[path] = amplitude_calibrator
p_animation = CameraAnimation(mapping)
n_events = df.index.size
if max_events is not None and n_events > max_events:
n_events = max_events
desc = "Looping over events"
for i, row in tqdm(df.iterrows(), total=n_events, desc=desc):
if i >= n_events:
break
hillas_path = row['path']
iev = row['iev']
iobs = row['iobs']
tduration = row['tduration']
reader = readers[hillas_path]
amplitude_calibrator = amplitude_calibrators[hillas_path]
waveforms = reader[iev]
shifted = time_calibrator(waveforms)
extracted = charge_extractor.process(shifted)
charge = extracted['charge_onsky']
time = extracted['t_onsky']
photons = amplitude_calibrator(charge, np.arange(n_pixels))
pe = photons * 0.25
mask = obtain_cleaning_mask(geom, photons, time)
if not mask.any():
msg = f"No pixels survived cleaning for: RUN {iobs} IEV {iev}"
raise ValueError(msg)
photons_ma = np.ma.masked_array(photons, mask=~mask)
min_pixel = photons_ma.argmin()
max_pixel = photons_ma.argmax()
min_image = -4
max_image = 0.7 * pe.max()
min_gf = shifted[max_pixel, :20].min()
max_gf = shifted[max_pixel].max() * 0.8
st = int(np.min(time[mask]) - 3)
et = int(np.max(time[mask]) + 6)
st = st if st > 0 else 0
et = et if et < n_samples else n_samples
p_animation.set_meta(i, iobs, iev, tduration)
p_animation.set_image(pe, min_image, max_image)
p_animation.set_waveforms(shifted[:, st:et:4], min_gf, max_gf)
p_animation.animate(output_dir, interval=50)
def main():
runlist_path = get_astri_2019("d2019-04-23_nudges/bright_50pe/runlist.txt")
df_runlist = pd.read_csv(runlist_path, sep='\t')
output = get_astri_2019("d2019-04-23_nudges/bright_50pe/charge.h5")
with HDF5Writer(output) as writer:
mapping = None
for _, row in df_runlist.iterrows():
run = row['run']
nudge = int(row['nudge'])
path = join(dirname(runlist_path), f"Run{run:05d}_r1.tio")
reader = ReaderR1(path, max_events=500)
mapping = reader.mapping
kw = dict(
n_pixels=reader.n_pixels,
n_samples=reader.n_samples,
mapping=reader.mapping,
reference_pulse_path=reader.reference_pulse_path,
)
baseline_subtractor = BaselineSubtractor(reader)
time_calibrator = TimeCalibrator()
extractor_cc = CrossCorrelation(**kw)
extractor_onsky_calib = OnskyCalibExtractor(**kw)
extractor_onsky = OnskyExtractor(**kw)
common = Common(**kw, _disable_by_default=True, waveform_max=True)
pixel_array = np.arange(reader.n_pixels)
monitor = get_nudge_and_temperature_from_reader(reader)
nudge_from_dac, temperature = monitor
assert nudge == nudge_from_dac
desc = "Looping over file"
for wfs in tqdm(reader, total=reader.n_events, desc=desc):
iev = wfs.iev
if wfs.stale.any():
continue
wfs = time_calibrator(wfs)
wfs = baseline_subtractor.subtract(wfs)
cc = extractor_cc.process(wfs)['charge_cc']
onsky_calib = extractor_onsky_calib.process(
wfs)['charge_onskycalib']
onsky = extractor_onsky.process(wfs)['charge_onsky']
waveform_max = common.process(wfs)['waveform_max']
params = dict(
nudge=nudge,
nudge_from_dac=nudge_from_dac,
temperature=temperature,
iev=iev,
pixel=pixel_array,
cc=cc,
onsky_calib=onsky_calib,
onsky=onsky,
waveform_max=waveform_max,
)
df = pd.DataFrame(params)
writer.append(df, key='data')
writer.add_mapping(mapping)