class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-image-sum-display"
infile = Unicode(
help='input simtelarray file',
default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag(
config=True)
telgroup = Integer(help='telescope group number',
default=1).tag(config=True)
aliases = Dict({
'infile': 'ImageSumDisplayerTool.infile',
'telgroup': 'ImageSumDisplayerTool.telgroup'
})
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
data = next(hessio_event_source(self.infile, max_events=1))
camtypes = get_camera_types(data.inst)
group = camtypes.groups[self.telgroup]
self._selected_tels = group['tel_id'].data
self._base_tel = self._selected_tels[0]
self.log.info("Telescope group %d: %s with %s camera", self.telgroup,
group[0]['tel_type'], group[0]['cam_type'])
self.log.info("SELECTED TELESCOPES:{}".format(self._selected_tels))
def start(self):
geom = None
imsum = None
disp = None
for data in hessio_event_source(self.infile,
allowed_tels=self._selected_tels,
max_events=None):
if geom is None:
x, y = data.inst.pixel_pos[self._base_tel]
flen = data.inst.optical_foclen[self._base_tel]
geom = CameraGeometry.guess(x, y, flen)
imsum = np.zeros(shape=x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.cam_id)
disp.add_colorbar()
disp.cmap = 'viridis'
if len(data.r0.tels_with_data) <= 2:
continue
imsum[:] = 0
for telid in data.r0.tels_with_data:
imsum += data.r0.tel[telid].adc_sums[0]
self.log.info("event={} ntels={} energy={}" \
.format(data.r0.event_id,
len(data.r0.tels_with_data),
data.mc.energy))
disp.image = imsum
plt.pause(0.1)
class MakeHistFirstCap(Tool):
name = "make-hist-first-cap"
infile = Unicode(help='input LST file', default="").tag(config=True)
max_events = Integer(help='stop after this many events if non-zero',
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(help='suffix (file extension) of output '
'filenames to write images '
'to (no writing is done if blank). '
'Images will be named [EVENTID][suffix]',
default_value="").tag(config=True)
aliases = Dict({
'infile': 'MakeHistFirstCap.infile',
'max-events': 'MakeHistFirstCap.max_events',
'output-suffix': 'MakeHistFirstCap.output_suffix'
})
def setup(self):
# load LST data
self.log.info("Read file:{}".format(self.infile))
self.reader = LSTEventSource(input_url=self.infile,
max_events=self.max_events)
def start(self):
fc = []
for event in self.reader:
fc.extend(get_first_capacitor_array(event))
plt.figure()
plt.hist(fc, bins=4096)
plt.ylabel("Number")
plt.xlabel("Stop Cell")
plt.show()
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Path(
help="input simtelarray file",
default_value=get_dataset_path("gamma_test_large.simtel.gz"),
exists=True,
directory_ok=False,
).tag(config=True)
telgroup = Integer(help="telescope group number", default_value=1).tag(config=True)
max_events = Integer(
help="stop after this many events if non-zero", default_value=0, min=0
).tag(config=True)
output_suffix = Unicode(
help="suffix (file extension) of output "
"filenames to write images "
"to (no writing is done if blank). "
"Images will be named [EVENTID][suffix]",
default_value="",
).tag(config=True)
aliases = Dict(
{
"infile": "ImageSumDisplayerTool.infile",
"telgroup": "ImageSumDisplayerTool.telgroup",
"max-events": "ImageSumDisplayerTool.max_events",
"output-suffix": "ImageSumDisplayerTool.output_suffix",
}
)
classes = [CameraCalibrator, SimTelEventSource]
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper instrument module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(
input_url=self.infile, max_events=self.max_events, back_seekable=True
)
camtypes = self.reader.subarray.to_table().group_by("camera_type")
self.reader.subarray.info(printer=self.log.info)
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group["tel_id"].data)
self._base_tel = self._selected_tels[0]
self.log.info(
"Telescope group %d: %s",
self.telgroup,
str(self.reader.subarray.tel[self._selected_tels[0]]),
)
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self, subarray=self.reader.subarray)
self.reader = SimTelEventSource(
input_url=self.infile,
max_events=self.max_events,
back_seekable=True,
allowed_tels=set(self._selected_tels),
)
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = self.reader.subarray.tel[self._base_tel].camera.geometry
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float64)
disp = CameraDisplay(geom, title=geom.camera_name)
disp.add_colorbar()
disp.cmap = "viridis"
if len(event.dl0.tel.keys()) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tel.keys():
imsum += event.dl1.tel[telid].image
self.log.info(
"event={} ntels={} energy={}".format(
event.index.event_id,
len(event.dl0.tel.keys()),
event.simulation.shower.energy,
)
)
disp.image = imsum
plt.pause(0.1)
if self.output_suffix != "":
filename = "{:020d}{}".format(event.index.event_id, self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
class SingleTelEventDisplay(Tool):
name = "ctapipe-display-televents"
description = Unicode(__doc__)
infile = Unicode(help="input file to read", default='').tag(config=True)
tel = Int(help='Telescope ID to display', default=0).tag(config=True)
channel = Integer(help="channel number to display", min=0,
max=1).tag(config=True)
write = Bool(help="Write out images to PNG files",
default=False).tag(config=True)
clean = Bool(help="Apply image cleaning", default=False).tag(config=True)
hillas = Bool(help="Apply and display Hillas parametrization",
default=False).tag(config=True)
samples = Bool(help="Show each sample", default=False).tag(config=True)
display = Bool(help="Display results in interactive window",
default_value=True).tag(config=True)
delay = Float(help='delay between events in s',
default_value=0.01,
min=0.001).tag(config=True)
progress = Bool(help='display progress bar',
default_value=True).tag(config=True)
aliases = Dict({
'infile': 'SingleTelEventDisplay.infile',
'tel': 'SingleTelEventDisplay.tel',
'max-events': 'EventSource.max_events',
'channel': 'SingleTelEventDisplay.channel',
'write': 'SingleTelEventDisplay.write',
'clean': 'SingleTelEventDisplay.clean',
'hillas': 'SingleTelEventDisplay.hillas',
'samples': 'SingleTelEventDisplay.samples',
'display': 'SingleTelEventDisplay.display',
'delay': 'SingleTelEventDisplay.delay',
'progress': 'SingleTelEventDisplay.progress'
})
classes = List([EventSource, CameraCalibrator])
def __init__(self, **kwargs):
super().__init__(**kwargs)
def setup(self):
print('TOLLES INFILE', self.infile)
self.event_source = EventSource.from_url(self.infile, parent=self)
self.event_source.allowed_tels = {
self.tel,
}
self.calibrator = CameraCalibrator(parent=self)
self.log.info(f'SELECTING EVENTS FROM TELESCOPE {self.tel}')
def start(self):
disp = None
for event in tqdm(self.event_source,
desc=f'Tel{self.tel}',
total=self.event_source.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug(f"Energy: {event.mc.energy}")
self.calibrator(event)
if disp is None:
geom = event.inst.subarray.tel[self.tel].camera
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:10d}'
f'_S{ii:02d}.png')
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color='pink',
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f'CT{self.tel:03d}_EV{event.r0.event_id:010d}.png')
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
'No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'.format(
self.tel, self.infile), )
class ImageSumDisplayerTool(Tool):
description = Unicode(__doc__)
name = "ctapipe-display-imagesum"
infile = Unicode(
help='input simtelarray file',
default="/Users/kosack/Data/CTA/Prod3/gamma.simtel.gz").tag(
config=True)
telgroup = Integer(help='telescope group number',
default=1).tag(config=True)
max_events = Integer(help='stop after this many events if non-zero',
default_value=0,
min=0).tag(config=True)
output_suffix = Unicode(help='suffix (file extension) of output '
'filenames to write images '
'to (no writing is done if blank). '
'Images will be named [EVENTID][suffix]',
default_value="").tag(config=True)
aliases = Dict({
'infile': 'ImageSumDisplayerTool.infile',
'telgroup': 'ImageSumDisplayerTool.telgroup',
'max-events': 'ImageSumDisplayerTool.max_events',
'output-suffix': 'ImageSumDisplayerTool.output_suffix'
})
classes = List([CameraCalibrator, SimTelEventSource])
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(input_url=self.infile,
max_events=self.max_events,
back_seekable=True)
for event in self.reader:
camtypes = event.inst.subarray.to_table().group_by('camera_type')
event.inst.subarray.info(printer=self.log.info)
break
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group['tel_id'].data)
self._base_tel = self._selected_tels[0]
self.log.info("Telescope group %d: %s", self.telgroup,
str(event.inst.subarray.tel[self._selected_tels[0]]))
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self)
self.reader.allowed_tels = self._selected_tels
def start(self):
geom = None
imsum = None
disp = None
for event in self.reader:
self.calibrator(event)
if geom is None:
geom = event.inst.subarray.tel[self._base_tel].camera
imsum = np.zeros(shape=geom.pix_x.shape, dtype=np.float)
disp = CameraDisplay(geom, title=geom.cam_id)
disp.add_colorbar()
disp.cmap = 'viridis'
if len(event.dl0.tels_with_data) <= 2:
continue
imsum[:] = 0
for telid in event.dl0.tels_with_data:
imsum += event.dl1.tel[telid].image
self.log.info("event={} ntels={} energy={}".format(
event.r0.event_id, len(event.dl0.tels_with_data),
event.mc.energy))
disp.image = imsum
plt.pause(0.1)
if self.output_suffix is not "":
filename = "{:020d}{}".format(event.r0.event_id,
self.output_suffix)
self.log.info(f"saving: '{filename}'")
plt.savefig(filename)
class DRS4PedestalAndSpikeHeight(Tool):
name = 'lstchain_create_drs4_pedestal_file'
output_path = Path(
directory_ok=False,
help=
'Path for the output hdf5 file of pedestal baseline and spike heights',
).tag(config=True)
skip_samples_front = Integer(
default_value=10,
help='Do not include first N samples in pedestal calculation').tag(
config=True)
skip_samples_end = Integer(
default_value=1,
help='Do not include last N samples in pedestal calculation').tag(
config=True)
progress_bar = Bool(help="Show progress bar during processing",
default_value=True).tag(config=True)
full_statistics = Bool(help=(
"If True, write spike{1,2,3} mean, count, std for each capacitor."
" Otherwise, only mean spike height for each gain, pixel is written"),
default_value=False).tag(config=True)
overwrite = Bool(help=("If true, overwrite output without asking,"
" else fail if output file already exists"),
default_value=False).tag(config=True)
aliases = {
('i', 'input'): 'LSTEventSource.input_url',
('o', 'output'): 'DRS4PedestalAndSpikeHeight.output_path',
('m', 'max-events'): 'LSTEventSource.max_events',
}
flags = {
**flag(
"overwrite",
"DRS4PedestalAndSpikeHeight.overwrite",
"Overwrite output file if it exists",
"Fail if output file already exists",
),
**flag(
"progress",
"DRS4PedestalAndSpikeHeight.progress_bar",
"Show a progress bar during event processing",
"Do not show a progress bar during event processing",
),
**flag(
"full-statistics",
"DRS4PedestalAndSpikeHeight.full_statistics",
"Whether to write the full statistics about spikes or not",
),
}
classes = [LSTEventSource]
def setup(self):
self.output_path = self.output_path.expanduser().resolve()
if self.output_path.exists():
if self.overwrite:
self.log.warning("Overwriting %s", self.output_path)
self.output_path.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_path} exists"
", use the `overwrite` option or choose another `output_path` "
)
self.log.debug("output path: %s", self.output_path)
Provenance().add_output_file(str(self.output_path), role="DL1/Event")
self.source = LSTEventSource(
parent=self,
pointing_information=False,
trigger_information=False,
)
# set some config options, these are necessary for this tool,
# so we set them here and not via the config system
self.source.r0_r1_calibrator.r1_sample_start = 0
self.source.r0_r1_calibrator.r1_sample_end = N_SAMPLES
self.source.r0_r1_calibrator.offset = 0
self.source.r0_r1_calibrator.apply_spike_correction = False
self.source.r0_r1_calibrator.apply_timelapse_correction = True
self.source.r0_r1_calibrator.apply_drs4_pedestal_correction = False
n_stats = N_GAINS * N_PIXELS * N_CAPACITORS_PIXEL
self.baseline_stats = OnlineStats(n_stats)
self.spike0_stats = OnlineStats(n_stats)
self.spike1_stats = OnlineStats(n_stats)
self.spike2_stats = OnlineStats(n_stats)
def start(self):
tel_id = self.source.tel_id
for event in tqdm(self.source, disable=not self.progress_bar):
fill_stats(
event.r1.tel[tel_id].waveform,
self.source.r0_r1_calibrator.first_cap[tel_id],
self.source.r0_r1_calibrator.first_cap_old[tel_id],
self.source.r0_r1_calibrator.last_readout_time[tel_id],
self.baseline_stats,
self.spike0_stats,
self.spike1_stats,
self.spike2_stats,
skip_samples_front=self.skip_samples_front,
skip_samples_end=self.skip_samples_end,
)
def mean_spike_height(self):
'''Calculate mean spike height for each gain, pixel'''
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
mean_baseline = self.baseline_stats.mean.reshape(shape)
spike_heights = np.full((N_GAINS, N_PIXELS, 3),
np.nan,
dtype=np.float32)
for i in range(3):
stats = getattr(self, f'spike{i}_stats')
counts = stats.counts.reshape(shape)
spike_height = stats.mean.reshape(shape) - mean_baseline
spike_height[counts == 0] = 0
# np.ma does not raise an error if the weights sum to 0
mean_height = np.ma.average(spike_height, weights=counts, axis=2)
# convert masked array to dense, replacing invalid values with nan
spike_heights[:, :, i] = mean_height.filled(np.nan)
unknown_spike_heights = np.isnan(spike_heights).any(axis=2)
n_unknown_spike_heights = np.count_nonzero(unknown_spike_heights)
if n_unknown_spike_heights > 0:
self.log.warning(
f'Could not determine spike height for {n_unknown_spike_heights} channels'
)
self.log.warning(
f'Gain, pixel: {np.nonzero(unknown_spike_heights)}')
# replace any unknown pixels with the mean over the camera
camera_mean_spike_height = np.nanmean(spike_heights, axis=(0, 1))
self.log.warning(
f'Using camera mean of {camera_mean_spike_height} for these pixels'
)
spike_heights[unknown_spike_heights] = camera_mean_spike_height
return spike_heights
def finish(self):
tel_id = self.source.tel_id
self.log.info('Writing output to %s', self.output_path)
key = f'r1/monitoring/drs4_baseline/tel_{tel_id:03d}'
shape = (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
baseline_mean = self.baseline_stats.mean.reshape(shape)
baseline_std = self.baseline_stats.std.reshape(shape)
baseline_counts = self.baseline_stats.counts.reshape(shape).astype(
np.uint16)
n_negative = np.count_nonzero(baseline_mean < 0)
if n_negative > 0:
gain, pixel, capacitor = np.nonzero(baseline_mean < 0)
self.log.critical(
f'{n_negative} baseline values are smaller than 0')
self.log.info("Gain | Pixel | Capacitor | Baseline ")
for g, p, c in zip(gain, pixel, capacitor):
self.log.info(
f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
)
n_small = np.count_nonzero(baseline_mean < 25)
if n_small > 0:
gain, pixel, capacitor = np.nonzero(baseline_mean < 25)
self.log.warning(f'{n_small} baseline values are smaller than 25')
self.log.info("Gain | Pixel | Capacitor | Baseline ")
for g, p, c in zip(gain, pixel, capacitor):
self.log.info(
f"{g:4d} | {p:4d} | {c:9d} | {baseline_mean[g][p][c]:6.1f}"
)
# Convert baseline mean and spike heights to uint16, handle missing
# values and values smaller 0, larger maxint
baseline_mean = convert_to_uint16(baseline_mean)
baseline_std = convert_to_uint16(baseline_std)
spike_height = convert_to_uint16(self.mean_spike_height())
with HDF5TableWriter(self.output_path) as writer:
Provenance().add_output_file(str(self.output_path))
drs4_calibration = DRS4CalibrationContainer(
baseline_mean=baseline_mean,
baseline_std=baseline_std,
baseline_counts=baseline_counts,
spike_height=spike_height,
)
writer.write(key, drs4_calibration)
