def test_pipeline():
dataset = get_dataset_path("chec_r1.tio")
reader = TargetIOEventSource(input_url=dataset, max_events=10)
calibrator = CameraCalibrator(eventsource=reader)
for event in reader:
calibrator.calibrate(event)
assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_pipeline():
from ctapipe.io.sst1meventsource import SST1MEventSource
reader = SST1MEventSource(input_url=example_file_path, max_events=10)
calibrator = CameraCalibrator(eventsource=reader)
for event in reader:
calibrator.calibrate(event)
assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_pipeline():
from ctapipe.io.sst1meventsource import SST1MEventSource
reader = SST1MEventSource(input_url=example_file_path, max_events=10)
calibrator = CameraCalibrator(eventsource=reader)
for event in reader:
calibrator.calibrate(event)
assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_integration_correction_no_ref_pulse(example_event):
telid = list(example_event.r0.tel)[0]
delattr(example_event, 'mc')
calibrator = CameraCalibrator()
calibrator._calibrate_dl0(example_event, telid)
correction = calibrator._get_correction(example_event, telid)
assert correction[0] == 1
def test_config(example_subarray):
calibrator = CameraCalibrator(subarray=example_subarray)
# test defaults
assert isinstance(calibrator.image_extractor, NeighborPeakWindowSum)
assert isinstance(calibrator.data_volume_reducer, NullDataVolumeReducer)
config = Config({
"CameraCalibrator": {
"image_extractor_type": "LocalPeakWindowSum",
"LocalPeakWindowSum": {
"window_width": 15
},
"data_volume_reducer_type": "TailCutsDataVolumeReducer",
"TailCutsDataVolumeReducer": {
"TailcutsImageCleaner": {
"picture_threshold_pe": 20.0
}
},
}
})
calibrator = CameraCalibrator(example_subarray, config=config)
assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
assert calibrator.image_extractor.window_width.tel[None] == 15
assert isinstance(calibrator.data_volume_reducer,
TailCutsDataVolumeReducer)
assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[
None] == 20
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
reader_factory = EventFileReaderFactory(**kwargs)
reader_class = reader_factory.get_class()
self.reader = reader_class(**kwargs)
self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs)
first_event = self.reader.get_event(0)
self.geometry = Geometry(self.config, self, first_event)
self.amp_cut = {"LSTCam": 92.7,
"NectarCam": 90.6,
"FlashCam": 90.6,
"CHEC": 29.3}
self.dist_cut = {"LSTCam": 1.74 * u.deg,
"NectarCam": 3. * u.deg,
"FlashCam": 3. * u.deg,
"CHEC": 3.55 * u.deg}
self.tail_cut = {"LSTCam": (8, 16),
"NectarCam": (7, 14),
"FlashCam": (7, 14),
"CHEC": (3, 6)}
self.pix_cut = {"LSTCam": 5,
"NectarCam": 4,
"FlashCam": 4,
"CHEC": 4}
def test_pipeline():
dataset = get_dataset_path("chec_r1.tio")
reader = TargetIOEventSource(input_url=dataset, max_events=10)
calibrator = CameraCalibrator(eventsource=reader)
for event in reader:
calibrator.calibrate(event)
assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_integration_correction_no_ref_pulse(example_event):
telid = list(example_event.r0.tel)[0]
delattr(example_event, "mc")
calibrator = CameraCalibrator()
calibrator._calibrate_dl0(example_event, telid)
correction = calibrator._get_correction(example_event, telid)
assert (correction == 1).all()
def test_config(example_subarray):
calibrator = CameraCalibrator(subarray=example_subarray)
# test defaults
assert len(calibrator.image_extractors) == 1
assert isinstance(calibrator.image_extractors["NeighborPeakWindowSum"],
NeighborPeakWindowSum)
assert isinstance(calibrator.data_volume_reducer, NullDataVolumeReducer)
# test we can configure different extractors with different options
# per telescope.
config = Config({
"CameraCalibrator": {
"image_extractor_type": [
("type", "*", "GlobalPeakWindowSum"),
("id", 1, "LocalPeakWindowSum"),
],
"LocalPeakWindowSum": {
"window_width": 15
},
"GlobalPeakWindowSum": {
"window_width": [("type", "*", 10), ("id", 2, 8)]
},
"data_volume_reducer_type":
"TailCutsDataVolumeReducer",
"TailCutsDataVolumeReducer": {
"TailcutsImageCleaner": {
"picture_threshold_pe": 20.0
}
},
}
})
calibrator = CameraCalibrator(example_subarray, config=config)
assert "GlobalPeakWindowSum" in calibrator.image_extractors
assert "LocalPeakWindowSum" in calibrator.image_extractors
assert isinstance(calibrator.image_extractors["LocalPeakWindowSum"],
LocalPeakWindowSum)
assert isinstance(calibrator.image_extractors["GlobalPeakWindowSum"],
GlobalPeakWindowSum)
extractor_1 = calibrator.image_extractors[
calibrator.image_extractor_type.tel[1]]
assert isinstance(extractor_1, LocalPeakWindowSum)
assert extractor_1.window_width.tel[1] == 15
extractor_2 = calibrator.image_extractors[
calibrator.image_extractor_type.tel[2]]
assert isinstance(extractor_2, GlobalPeakWindowSum)
assert extractor_2.window_width.tel[2] == 8
extractor_3 = calibrator.image_extractors[
calibrator.image_extractor_type.tel[3]]
assert isinstance(extractor_3, GlobalPeakWindowSum)
assert extractor_3.window_width.tel[3] == 10
assert isinstance(calibrator.data_volume_reducer,
TailCutsDataVolumeReducer)
assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[
None] == 20
def test_check_r1_empty(example_event):
calibrator = CameraCalibrator()
telid = list(example_event.r0.tel)[0]
waveform = example_event.r1.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.r1.tel[telid].waveform = None
calibrator._calibrate_dl0(example_event, telid)
assert example_event.dl0.tel[telid].waveform == None
assert calibrator._check_r1_empty(None) is True
assert calibrator._check_r1_empty(waveform) is False
def test_dl1_charge_calib(example_subarray):
camera = CameraGeometry.from_name("CHEC")
n_pixels = camera.n_pixels
n_samples = 96
mid = n_samples // 2
pulse_sigma = 6
random = np.random.RandomState(1)
x = np.arange(n_samples)
# Randomize times and create pulses
time_offset = random.uniform(mid - 10, mid + 10, n_pixels)[:, np.newaxis]
y = norm.pdf(x, time_offset, pulse_sigma).astype("float32")
# Define absolute calibration coefficients
absolute = random.uniform(100, 1000, n_pixels).astype("float32")
y *= absolute[:, np.newaxis]
# Define relative coefficients
relative = random.normal(1, 0.01, n_pixels)
y /= relative[:, np.newaxis]
# Define pedestal
pedestal = random.uniform(-4, 4, n_pixels)
y += pedestal[:, np.newaxis]
event = DataContainer()
telid = list(example_subarray.tel.keys())[0]
event.dl0.tel[telid].waveform = y
# Test default
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=FullWaveformSum(subarray=example_subarray),
)
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, y.sum(1), rtol=1e-4)
event.calibration.tel[telid].dl1.time_shift = time_offset
event.calibration.tel[telid].dl1.pedestal_offset = pedestal * n_samples
event.calibration.tel[telid].dl1.absolute_factor = absolute
event.calibration.tel[telid].dl1.relative_factor = relative
# Test without need for timing corrections
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=FullWaveformSum(subarray=example_subarray),
)
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-5)
def test_manual_extractor(example_subarray):
extractor = LocalPeakWindowSum(subarray=example_subarray)
calibrator = CameraCalibrator(subarray=example_subarray,
image_extractor=extractor)
assert "LocalPeakWindowSum" in calibrator.image_extractors
assert calibrator.image_extractor_type.tel[1] == "LocalPeakWindowSum"
assert calibrator.image_extractors["LocalPeakWindowSum"] is extractor
def test_view_wf(example_event):
viewer = BokehEventViewer()
viewer.create()
viewer.event = example_event
c = CameraCalibrator()
c.calibrate(example_event)
t = list(example_event.r0.tels_with_data)[0]
wf = viewer.waveforms[0]
wf.view = 'r1'
assert (wf.waveform == example_event.r1.tel[t].waveform[0, 0, :]).all()
with pytest.raises(ValueError):
wf.view = 'q'
def test_view_camera(example_event):
viewer = BokehEventViewer()
viewer.create()
viewer.event = example_event
c = CameraCalibrator()
c.calibrate(example_event)
t = list(example_event.r0.tels_with_data)[0]
cam = viewer.cameras[0]
cam.view = 'r1'
assert (cam.image == example_event.r1.tel[t].waveform[0, :, 0]).all()
with pytest.raises(ValueError):
cam.view = 'q'
def test_camera_calibrator(example_event):
telid = list(example_event.r0.tel)[0]
calibrator = CameraCalibrator(subarray=example_event.inst.subarray)
calibrator(example_event)
image = example_event.dl1.tel[telid].image
pulse_time = example_event.dl1.tel[telid].pulse_time
assert image is not None
assert pulse_time is not None
assert image.shape == (1764, )
assert pulse_time.shape == (1764, )
def test_check_dl0_empty(example_event):
calibrator = CameraCalibrator()
telid = list(example_event.r0.tel)[0]
calibrator._calibrate_dl0(example_event, telid)
waveform = example_event.dl0.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.dl0.tel[telid].waveform = None
calibrator._calibrate_dl1(example_event, telid)
assert example_event.dl1.tel[telid].image is None
assert calibrator._check_dl0_empty(None) is True
assert calibrator._check_dl0_empty(waveform) is False
calibrator = CameraCalibrator()
event = DataContainer()
event.dl1.tel[telid].image = np.full(2048, 2)
with pytest.warns(UserWarning):
calibrator(event)
assert (event.dl1.tel[telid].image == 2).all()
def test_config():
window_shift = 3
window_width = 9
config = Config({
"LocalPeakWindowSum": {
"window_shift": window_shift,
"window_width": window_width,
}
})
calibrator = CameraCalibrator(
image_extractor=LocalPeakWindowSum(config=config), config=config)
assert calibrator.image_extractor.window_shift.tel[None] == window_shift
assert calibrator.image_extractor.window_width.tel[None] == window_width
def test_view_wf(example_event, example_subarray):
viewer = BokehEventViewer(example_subarray)
viewer.create()
viewer.event = example_event
calibrator = CameraCalibrator(subarray=example_subarray)
calibrator(example_event)
t = list(example_event.r0.tel.keys())[0]
wf = viewer.waveforms[0]
wf.view = "r1"
assert (wf.waveform == example_event.r1.tel[t].waveform[0, :]).all()
with pytest.raises(ValueError):
wf.view = "q"
def test_view_camera(example_event, example_subarray):
viewer = BokehEventViewer(example_subarray)
viewer.create()
viewer.event = example_event
calibrator = CameraCalibrator(subarray=example_subarray)
calibrator(example_event)
t = list(example_event.r0.tel.keys())[0]
cam = viewer.cameras[0]
cam.view = "r1"
assert (cam.image == example_event.r1.tel[t].waveform[:, 0]).all()
with pytest.raises(ValueError):
cam.view = "q"
def test_check_r1_empty(example_event):
calibrator = CameraCalibrator()
telid = list(example_event.r0.tel)[0]
waveform = example_event.r1.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.r1.tel[telid].waveform = None
calibrator._calibrate_dl0(example_event, telid)
assert example_event.dl0.tel[telid].waveform == None
assert calibrator._check_r1_empty(None) is True
assert calibrator._check_r1_empty(waveform) is False
def test_check_r1_empty(example_event):
calibrator = CameraCalibrator()
telid = list(example_event.r0.tel)[0]
waveform = example_event.r1.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.r1.tel[telid].waveform = None
calibrator._calibrate_dl0(example_event, telid)
assert example_event.dl0.tel[telid].waveform is None
assert calibrator._check_r1_empty(None) is True
assert calibrator._check_r1_empty(waveform) is False
calibrator = CameraCalibrator(image_extractor=FullWaveformSum())
event = DataContainer()
event.dl0.tel[telid].waveform = np.full((2048, 128), 2)
with pytest.warns(UserWarning):
calibrator(event)
assert (event.dl0.tel[telid].waveform == 2).all()
assert (event.dl1.tel[telid].image == 2 * 128).all()
def test_config(example_subarray):
window_shift = 3
window_width = 9
config = Config(
{
"LocalPeakWindowSum": {
"window_shift": window_shift,
"window_width": window_width,
}
}
)
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=LocalPeakWindowSum(subarray=example_subarray, config=config),
config=config,
)
assert calibrator.image_extractor.window_shift.tel[None] == window_shift
assert calibrator.image_extractor.window_width.tel[None] == window_width
def test_select_gain():
n_channels = 2
n_pixels = 2048
n_samples = 128
telid = 0
calibrator = CameraCalibrator()
event = DataContainer()
event.r1.tel[telid].waveform = np.ones((n_channels, n_pixels, n_samples))
calibrator._calibrate_dl0(event, telid)
assert event.dl0.tel[telid].waveform.shape == (n_pixels, n_samples)
event = DataContainer()
event.r1.tel[telid].waveform = np.ones((n_pixels, n_samples))
with pytest.raises(ValueError):
calibrator._calibrate_dl0(event, telid)
event = DataContainer()
event.r1.tel[telid].waveform = np.ones((n_pixels, n_samples))
event.r1.tel[telid].selected_gain_channel = np.zeros(n_pixels)
calibrator._calibrate_dl0(event, telid)
assert event.dl0.tel[telid].waveform.shape == (n_pixels, n_samples)
def test_camera_calibrator(example_event):
telid = list(example_event.r0.tel)[0]
calibrator = CameraCalibrator()
calibrator(example_event)
image = example_event.dl1.tel[telid].image
assert image is not None
def test_manual_extractor():
calibrator = CameraCalibrator(image_extractor=LocalPeakWindowSum())
assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
def test_manual_extractor(example_subarray):
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=LocalPeakWindowSum(subarray=example_subarray))
assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
class EventFileLooper(Tool):
name = "EventFileLooper"
description = "Loop through the file and apply calibration. Intended as " \
"a test that the routines work, and a benchmark of speed."
aliases = Dict(dict(r='EventFileReaderFactory.reader',
f='EventFileReaderFactory.input_path',
max_events='EventFileReaderFactory.max_events',
ped='CameraR1CalibratorFactory.pedestal_path',
tf='CameraR1CalibratorFactory.tf_path',
pe='CameraR1CalibratorFactory.pe_path',
extractor='ChargeExtractorFactory.extractor',
extractor_t0='ChargeExtractorFactory.t0',
window_width='ChargeExtractorFactory.window_width',
window_shift='ChargeExtractorFactory.window_shift',
sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG',
sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG',
lwt='ChargeExtractorFactory.lwt',
clip_amplitude='CameraDL1Calibrator.clip_amplitude',
radius='CameraDL1Calibrator.radius',
cleaner='WaveformCleanerFactory.cleaner',
))
classes = List([EventFileReaderFactory,
ChargeExtractorFactory,
CameraR1CalibratorFactory,
CameraDL1Calibrator,
WaveformCleanerFactory
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reader = None
self.calibrator = None
self.geometry = None
self.amp_cut = None
self.dist_cut = None
self.tail_cut = None
self.pix_cut = None
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
reader_factory = EventFileReaderFactory(**kwargs)
reader_class = reader_factory.get_class()
self.reader = reader_class(**kwargs)
self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs)
first_event = self.reader.get_event(0)
self.geometry = Geometry(self.config, self, first_event)
self.amp_cut = {"LSTCam": 92.7,
"NectarCam": 90.6,
"FlashCam": 90.6,
"CHEC": 29.3}
self.dist_cut = {"LSTCam": 1.74 * u.deg,
"NectarCam": 3. * u.deg,
"FlashCam": 3. * u.deg,
"CHEC": 3.55 * u.deg}
self.tail_cut = {"LSTCam": (8, 16),
"NectarCam": (7, 14),
"FlashCam": (7, 14),
"CHEC": (3, 6)}
self.pix_cut = {"LSTCam": 5,
"NectarCam": 4,
"FlashCam": 4,
"CHEC": 4}
def start(self):
# n_events = self.reader.num_events
source = self.reader.read()
desc = "Looping through file"
for event in tqdm(source, desc=desc): #, total=n_events):
ev = event.count
self.calibrator.calibrate(event)
for tel_id in event.r0.tels_with_data:
geom = self.geometry.get_camera(tel_id)
nom_geom = self.geometry.get_nominal(tel_id)
image = event.dl1.tel[tel_id].image[0]
# Cleaning
cuts = self.tail_cut[geom.cam_id]
tc = tailcuts_clean(geom, image, *cuts)
if not tc.any():
# self.log.warning('No image')
continue
# cleaned = np.ma.masked_array(image, mask=~tc)
cleaned = image * tc
# Hillas
try:
hillas = hillas_parameters(nom_geom, cleaned)
except HillasParameterizationError:
# self.log.warning('HillasParameterizationError')
continue
# embed()
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
camera = CameraDisplay(nom_geom, ax=ax, image=image, cmap='viridis')
camera.add_colorbar()
cen_x = u.Quantity(hillas.cen_x).value
cen_y = u.Quantity(hillas.cen_y).value
length = u.Quantity(hillas.length).value
width = u.Quantity(hillas.width).value
print(cen_x, cen_y, length, width)
camera.add_ellipse(centroid=(cen_x, cen_y),
length=length * 2,
width=width * 2,
angle=hillas.psi.rad)
plt.show()
def finish(self):
pass
def test_dl1_charge_calib(example_subarray):
# copy because we mutate the camera, should not affect other tests
subarray = deepcopy(example_subarray)
camera = subarray.tel[1].camera
# test with a sampling_rate different than 1 to
# test if we handle time vs. slices correctly
sampling_rate = 2
camera.readout.sampling_rate = sampling_rate * u.GHz
n_pixels = camera.geometry.n_pixels
n_samples = 96
mid = n_samples // 2
pulse_sigma = 6
random = np.random.RandomState(1)
x = np.arange(n_samples)
# Randomize times and create pulses
time_offset = random.uniform(-10, +10, n_pixels)
y = norm.pdf(x, mid + time_offset[:, np.newaxis],
pulse_sigma).astype("float32")
camera.readout.reference_pulse_shape = norm.pdf(x, mid,
pulse_sigma)[np.newaxis, :]
camera.readout.reference_pulse_sample_width = 1 / camera.readout.sampling_rate
# Define absolute calibration coefficients
absolute = random.uniform(100, 1000, n_pixels).astype("float32")
y *= absolute[:, np.newaxis]
# Define relative coefficients
relative = random.normal(1, 0.01, n_pixels)
y /= relative[:, np.newaxis]
# Define pedestal
pedestal = random.uniform(-4, 4, n_pixels)
y += pedestal[:, np.newaxis]
event = ArrayEventContainer()
telid = list(subarray.tel.keys())[0]
event.dl0.tel[telid].waveform = y
event.dl0.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int)
event.r1.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int)
# Test default
calibrator = CameraCalibrator(
subarray=subarray, image_extractor=FullWaveformSum(subarray=subarray))
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, y.sum(1), rtol=1e-4)
event.calibration.tel[telid].dl1.pedestal_offset = pedestal
event.calibration.tel[telid].dl1.absolute_factor = absolute
event.calibration.tel[telid].dl1.relative_factor = relative
# Test without timing corrections
calibrator(event)
dl1 = event.dl1.tel[telid]
np.testing.assert_allclose(dl1.image, 1, rtol=1e-5)
expected_peak_time = (mid + time_offset) / sampling_rate
np.testing.assert_allclose(dl1.peak_time, expected_peak_time, rtol=1e-5)
# test with timing corrections
event.calibration.tel[telid].dl1.time_shift = time_offset / sampling_rate
calibrator(event)
# more rtol since shifting might lead to reduced integral
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-5)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
mid / sampling_rate,
atol=1)
# test not applying time shifts
# now we should be back to the result without setting time shift
calibrator.apply_peak_time_shift = False
calibrator.apply_waveform_time_shift = False
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-4)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
expected_peak_time,
atol=1)
# We now use GlobalPeakWindowSum to see the effect of missing charge
# due to not correcting time offsets.
calibrator = CameraCalibrator(
subarray=subarray,
image_extractor=GlobalPeakWindowSum(subarray=subarray))
calibrator(event)
# test with timing corrections, should work
# higher rtol because we cannot shift perfectly
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=0.01)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
mid / sampling_rate,
atol=1)
# test deactivating timing corrections
calibrator.apply_waveform_time_shift = False
calibrator(event)
# make sure we chose an example where the time shifts matter
# charges should be quite off due to summing around global shift
assert not np.allclose(event.dl1.tel[telid].image, 1, rtol=0.1)
assert not np.allclose(
event.dl1.tel[telid].peak_time, mid / sampling_rate, atol=1)
