def test_pipeline(): dataset = get_dataset_path("chec_r1.tio") reader = TargetIOEventSource(input_url=dataset, max_events=10) calibrator = CameraCalibrator(eventsource=reader) for event in reader: calibrator.calibrate(event) assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_pipeline(): from ctapipe.io.sst1meventsource import SST1MEventSource reader = SST1MEventSource(input_url=example_file_path, max_events=10) calibrator = CameraCalibrator(eventsource=reader) for event in reader: calibrator.calibrate(event) assert event.r0.tel.keys() == event.dl1.tel.keys()
def test_integration_correction_no_ref_pulse(example_event): telid = list(example_event.r0.tel)[0] delattr(example_event, 'mc') calibrator = CameraCalibrator() calibrator._calibrate_dl0(example_event, telid) correction = calibrator._get_correction(example_event, telid) assert correction[0] == 1
def test_config(example_subarray): calibrator = CameraCalibrator(subarray=example_subarray) # test defaults assert isinstance(calibrator.image_extractor, NeighborPeakWindowSum) assert isinstance(calibrator.data_volume_reducer, NullDataVolumeReducer) config = Config({ "CameraCalibrator": { "image_extractor_type": "LocalPeakWindowSum", "LocalPeakWindowSum": { "window_width": 15 }, "data_volume_reducer_type": "TailCutsDataVolumeReducer", "TailCutsDataVolumeReducer": { "TailcutsImageCleaner": { "picture_threshold_pe": 20.0 } }, } }) calibrator = CameraCalibrator(example_subarray, config=config) assert isinstance(calibrator.image_extractor, LocalPeakWindowSum) assert calibrator.image_extractor.window_width.tel[None] == 15 assert isinstance(calibrator.data_volume_reducer, TailCutsDataVolumeReducer) assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[ None] == 20
def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) first_event = self.reader.get_event(0) self.geometry = Geometry(self.config, self, first_event) self.amp_cut = {"LSTCam": 92.7, "NectarCam": 90.6, "FlashCam": 90.6, "CHEC": 29.3} self.dist_cut = {"LSTCam": 1.74 * u.deg, "NectarCam": 3. * u.deg, "FlashCam": 3. * u.deg, "CHEC": 3.55 * u.deg} self.tail_cut = {"LSTCam": (8, 16), "NectarCam": (7, 14), "FlashCam": (7, 14), "CHEC": (3, 6)} self.pix_cut = {"LSTCam": 5, "NectarCam": 4, "FlashCam": 4, "CHEC": 4}
def test_integration_correction_no_ref_pulse(example_event): telid = list(example_event.r0.tel)[0] delattr(example_event, "mc") calibrator = CameraCalibrator() calibrator._calibrate_dl0(example_event, telid) correction = calibrator._get_correction(example_event, telid) assert (correction == 1).all()
def test_config(example_subarray): calibrator = CameraCalibrator(subarray=example_subarray) # test defaults assert len(calibrator.image_extractors) == 1 assert isinstance(calibrator.image_extractors["NeighborPeakWindowSum"], NeighborPeakWindowSum) assert isinstance(calibrator.data_volume_reducer, NullDataVolumeReducer) # test we can configure different extractors with different options # per telescope. config = Config({ "CameraCalibrator": { "image_extractor_type": [ ("type", "*", "GlobalPeakWindowSum"), ("id", 1, "LocalPeakWindowSum"), ], "LocalPeakWindowSum": { "window_width": 15 }, "GlobalPeakWindowSum": { "window_width": [("type", "*", 10), ("id", 2, 8)] }, "data_volume_reducer_type": "TailCutsDataVolumeReducer", "TailCutsDataVolumeReducer": { "TailcutsImageCleaner": { "picture_threshold_pe": 20.0 } }, } }) calibrator = CameraCalibrator(example_subarray, config=config) assert "GlobalPeakWindowSum" in calibrator.image_extractors assert "LocalPeakWindowSum" in calibrator.image_extractors assert isinstance(calibrator.image_extractors["LocalPeakWindowSum"], LocalPeakWindowSum) assert isinstance(calibrator.image_extractors["GlobalPeakWindowSum"], GlobalPeakWindowSum) extractor_1 = calibrator.image_extractors[ calibrator.image_extractor_type.tel[1]] assert isinstance(extractor_1, LocalPeakWindowSum) assert extractor_1.window_width.tel[1] == 15 extractor_2 = calibrator.image_extractors[ calibrator.image_extractor_type.tel[2]] assert isinstance(extractor_2, GlobalPeakWindowSum) assert extractor_2.window_width.tel[2] == 8 extractor_3 = calibrator.image_extractors[ calibrator.image_extractor_type.tel[3]] assert isinstance(extractor_3, GlobalPeakWindowSum) assert extractor_3.window_width.tel[3] == 10 assert isinstance(calibrator.data_volume_reducer, TailCutsDataVolumeReducer) assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[ None] == 20
def test_check_r1_empty(example_event): calibrator = CameraCalibrator() telid = list(example_event.r0.tel)[0] waveform = example_event.r1.tel[telid].waveform.copy() with pytest.warns(UserWarning): example_event.r1.tel[telid].waveform = None calibrator._calibrate_dl0(example_event, telid) assert example_event.dl0.tel[telid].waveform == None assert calibrator._check_r1_empty(None) is True assert calibrator._check_r1_empty(waveform) is False
def test_dl1_charge_calib(example_subarray): camera = CameraGeometry.from_name("CHEC") n_pixels = camera.n_pixels n_samples = 96 mid = n_samples // 2 pulse_sigma = 6 random = np.random.RandomState(1) x = np.arange(n_samples) # Randomize times and create pulses time_offset = random.uniform(mid - 10, mid + 10, n_pixels)[:, np.newaxis] y = norm.pdf(x, time_offset, pulse_sigma).astype("float32") # Define absolute calibration coefficients absolute = random.uniform(100, 1000, n_pixels).astype("float32") y *= absolute[:, np.newaxis] # Define relative coefficients relative = random.normal(1, 0.01, n_pixels) y /= relative[:, np.newaxis] # Define pedestal pedestal = random.uniform(-4, 4, n_pixels) y += pedestal[:, np.newaxis] event = DataContainer() telid = list(example_subarray.tel.keys())[0] event.dl0.tel[telid].waveform = y # Test default calibrator = CameraCalibrator( subarray=example_subarray, image_extractor=FullWaveformSum(subarray=example_subarray), ) calibrator(event) np.testing.assert_allclose(event.dl1.tel[telid].image, y.sum(1), rtol=1e-4) event.calibration.tel[telid].dl1.time_shift = time_offset event.calibration.tel[telid].dl1.pedestal_offset = pedestal * n_samples event.calibration.tel[telid].dl1.absolute_factor = absolute event.calibration.tel[telid].dl1.relative_factor = relative # Test without need for timing corrections calibrator = CameraCalibrator( subarray=example_subarray, image_extractor=FullWaveformSum(subarray=example_subarray), ) calibrator(event) np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-5)
def test_manual_extractor(example_subarray): extractor = LocalPeakWindowSum(subarray=example_subarray) calibrator = CameraCalibrator(subarray=example_subarray, image_extractor=extractor) assert "LocalPeakWindowSum" in calibrator.image_extractors assert calibrator.image_extractor_type.tel[1] == "LocalPeakWindowSum" assert calibrator.image_extractors["LocalPeakWindowSum"] is extractor
def test_view_wf(example_event): viewer = BokehEventViewer() viewer.create() viewer.event = example_event c = CameraCalibrator() c.calibrate(example_event) t = list(example_event.r0.tels_with_data)[0] wf = viewer.waveforms[0] wf.view = 'r1' assert (wf.waveform == example_event.r1.tel[t].waveform[0, 0, :]).all() with pytest.raises(ValueError): wf.view = 'q'
def test_view_camera(example_event): viewer = BokehEventViewer() viewer.create() viewer.event = example_event c = CameraCalibrator() c.calibrate(example_event) t = list(example_event.r0.tels_with_data)[0] cam = viewer.cameras[0] cam.view = 'r1' assert (cam.image == example_event.r1.tel[t].waveform[0, :, 0]).all() with pytest.raises(ValueError): cam.view = 'q'
def test_camera_calibrator(example_event): telid = list(example_event.r0.tel)[0] calibrator = CameraCalibrator(subarray=example_event.inst.subarray) calibrator(example_event) image = example_event.dl1.tel[telid].image pulse_time = example_event.dl1.tel[telid].pulse_time assert image is not None assert pulse_time is not None assert image.shape == (1764, ) assert pulse_time.shape == (1764, )
def test_check_dl0_empty(example_event): calibrator = CameraCalibrator() telid = list(example_event.r0.tel)[0] calibrator._calibrate_dl0(example_event, telid) waveform = example_event.dl0.tel[telid].waveform.copy() with pytest.warns(UserWarning): example_event.dl0.tel[telid].waveform = None calibrator._calibrate_dl1(example_event, telid) assert example_event.dl1.tel[telid].image is None assert calibrator._check_dl0_empty(None) is True assert calibrator._check_dl0_empty(waveform) is False calibrator = CameraCalibrator() event = DataContainer() event.dl1.tel[telid].image = np.full(2048, 2) with pytest.warns(UserWarning): calibrator(event) assert (event.dl1.tel[telid].image == 2).all()
def test_config(): window_shift = 3 window_width = 9 config = Config({ "LocalPeakWindowSum": { "window_shift": window_shift, "window_width": window_width, } }) calibrator = CameraCalibrator( image_extractor=LocalPeakWindowSum(config=config), config=config) assert calibrator.image_extractor.window_shift.tel[None] == window_shift assert calibrator.image_extractor.window_width.tel[None] == window_width
def test_view_wf(example_event, example_subarray): viewer = BokehEventViewer(example_subarray) viewer.create() viewer.event = example_event calibrator = CameraCalibrator(subarray=example_subarray) calibrator(example_event) t = list(example_event.r0.tel.keys())[0] wf = viewer.waveforms[0] wf.view = "r1" assert (wf.waveform == example_event.r1.tel[t].waveform[0, :]).all() with pytest.raises(ValueError): wf.view = "q"
def test_view_camera(example_event, example_subarray): viewer = BokehEventViewer(example_subarray) viewer.create() viewer.event = example_event calibrator = CameraCalibrator(subarray=example_subarray) calibrator(example_event) t = list(example_event.r0.tel.keys())[0] cam = viewer.cameras[0] cam.view = "r1" assert (cam.image == example_event.r1.tel[t].waveform[:, 0]).all() with pytest.raises(ValueError): cam.view = "q"
def test_check_r1_empty(example_event): calibrator = CameraCalibrator() telid = list(example_event.r0.tel)[0] waveform = example_event.r1.tel[telid].waveform.copy() with pytest.warns(UserWarning): example_event.r1.tel[telid].waveform = None calibrator._calibrate_dl0(example_event, telid) assert example_event.dl0.tel[telid].waveform is None assert calibrator._check_r1_empty(None) is True assert calibrator._check_r1_empty(waveform) is False calibrator = CameraCalibrator(image_extractor=FullWaveformSum()) event = DataContainer() event.dl0.tel[telid].waveform = np.full((2048, 128), 2) with pytest.warns(UserWarning): calibrator(event) assert (event.dl0.tel[telid].waveform == 2).all() assert (event.dl1.tel[telid].image == 2 * 128).all()
def test_config(example_subarray): window_shift = 3 window_width = 9 config = Config( { "LocalPeakWindowSum": { "window_shift": window_shift, "window_width": window_width, } } ) calibrator = CameraCalibrator( subarray=example_subarray, image_extractor=LocalPeakWindowSum(subarray=example_subarray, config=config), config=config, ) assert calibrator.image_extractor.window_shift.tel[None] == window_shift assert calibrator.image_extractor.window_width.tel[None] == window_width
def test_select_gain(): n_channels = 2 n_pixels = 2048 n_samples = 128 telid = 0 calibrator = CameraCalibrator() event = DataContainer() event.r1.tel[telid].waveform = np.ones((n_channels, n_pixels, n_samples)) calibrator._calibrate_dl0(event, telid) assert event.dl0.tel[telid].waveform.shape == (n_pixels, n_samples) event = DataContainer() event.r1.tel[telid].waveform = np.ones((n_pixels, n_samples)) with pytest.raises(ValueError): calibrator._calibrate_dl0(event, telid) event = DataContainer() event.r1.tel[telid].waveform = np.ones((n_pixels, n_samples)) event.r1.tel[telid].selected_gain_channel = np.zeros(n_pixels) calibrator._calibrate_dl0(event, telid) assert event.dl0.tel[telid].waveform.shape == (n_pixels, n_samples)
def test_camera_calibrator(example_event): telid = list(example_event.r0.tel)[0] calibrator = CameraCalibrator() calibrator(example_event) image = example_event.dl1.tel[telid].image assert image is not None
def test_manual_extractor(): calibrator = CameraCalibrator(image_extractor=LocalPeakWindowSum()) assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
def test_manual_extractor(example_subarray): calibrator = CameraCalibrator( subarray=example_subarray, image_extractor=LocalPeakWindowSum(subarray=example_subarray)) assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
class EventFileLooper(Tool): name = "EventFileLooper" description = "Loop through the file and apply calibration. Intended as " \ "a test that the routines work, and a benchmark of speed." aliases = Dict(dict(r='EventFileReaderFactory.reader', f='EventFileReaderFactory.input_path', max_events='EventFileReaderFactory.max_events', ped='CameraR1CalibratorFactory.pedestal_path', tf='CameraR1CalibratorFactory.tf_path', pe='CameraR1CalibratorFactory.pe_path', extractor='ChargeExtractorFactory.extractor', extractor_t0='ChargeExtractorFactory.t0', window_width='ChargeExtractorFactory.window_width', window_shift='ChargeExtractorFactory.window_shift', sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', lwt='ChargeExtractorFactory.lwt', clip_amplitude='CameraDL1Calibrator.clip_amplitude', radius='CameraDL1Calibrator.radius', cleaner='WaveformCleanerFactory.cleaner', )) classes = List([EventFileReaderFactory, ChargeExtractorFactory, CameraR1CalibratorFactory, CameraDL1Calibrator, WaveformCleanerFactory ]) def __init__(self, **kwargs): super().__init__(**kwargs) self.reader = None self.calibrator = None self.geometry = None self.amp_cut = None self.dist_cut = None self.tail_cut = None self.pix_cut = None def setup(self): self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) reader_factory = EventFileReaderFactory(**kwargs) reader_class = reader_factory.get_class() self.reader = reader_class(**kwargs) self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs) first_event = self.reader.get_event(0) self.geometry = Geometry(self.config, self, first_event) self.amp_cut = {"LSTCam": 92.7, "NectarCam": 90.6, "FlashCam": 90.6, "CHEC": 29.3} self.dist_cut = {"LSTCam": 1.74 * u.deg, "NectarCam": 3. * u.deg, "FlashCam": 3. * u.deg, "CHEC": 3.55 * u.deg} self.tail_cut = {"LSTCam": (8, 16), "NectarCam": (7, 14), "FlashCam": (7, 14), "CHEC": (3, 6)} self.pix_cut = {"LSTCam": 5, "NectarCam": 4, "FlashCam": 4, "CHEC": 4} def start(self): # n_events = self.reader.num_events source = self.reader.read() desc = "Looping through file" for event in tqdm(source, desc=desc): #, total=n_events): ev = event.count self.calibrator.calibrate(event) for tel_id in event.r0.tels_with_data: geom = self.geometry.get_camera(tel_id) nom_geom = self.geometry.get_nominal(tel_id) image = event.dl1.tel[tel_id].image[0] # Cleaning cuts = self.tail_cut[geom.cam_id] tc = tailcuts_clean(geom, image, *cuts) if not tc.any(): # self.log.warning('No image') continue # cleaned = np.ma.masked_array(image, mask=~tc) cleaned = image * tc # Hillas try: hillas = hillas_parameters(nom_geom, cleaned) except HillasParameterizationError: # self.log.warning('HillasParameterizationError') continue # embed() fig = plt.figure(figsize=(10, 10)) ax = fig.add_subplot(111) camera = CameraDisplay(nom_geom, ax=ax, image=image, cmap='viridis') camera.add_colorbar() cen_x = u.Quantity(hillas.cen_x).value cen_y = u.Quantity(hillas.cen_y).value length = u.Quantity(hillas.length).value width = u.Quantity(hillas.width).value print(cen_x, cen_y, length, width) camera.add_ellipse(centroid=(cen_x, cen_y), length=length * 2, width=width * 2, angle=hillas.psi.rad) plt.show() def finish(self): pass
def test_dl1_charge_calib(example_subarray): # copy because we mutate the camera, should not affect other tests subarray = deepcopy(example_subarray) camera = subarray.tel[1].camera # test with a sampling_rate different than 1 to # test if we handle time vs. slices correctly sampling_rate = 2 camera.readout.sampling_rate = sampling_rate * u.GHz n_pixels = camera.geometry.n_pixels n_samples = 96 mid = n_samples // 2 pulse_sigma = 6 random = np.random.RandomState(1) x = np.arange(n_samples) # Randomize times and create pulses time_offset = random.uniform(-10, +10, n_pixels) y = norm.pdf(x, mid + time_offset[:, np.newaxis], pulse_sigma).astype("float32") camera.readout.reference_pulse_shape = norm.pdf(x, mid, pulse_sigma)[np.newaxis, :] camera.readout.reference_pulse_sample_width = 1 / camera.readout.sampling_rate # Define absolute calibration coefficients absolute = random.uniform(100, 1000, n_pixels).astype("float32") y *= absolute[:, np.newaxis] # Define relative coefficients relative = random.normal(1, 0.01, n_pixels) y /= relative[:, np.newaxis] # Define pedestal pedestal = random.uniform(-4, 4, n_pixels) y += pedestal[:, np.newaxis] event = ArrayEventContainer() telid = list(subarray.tel.keys())[0] event.dl0.tel[telid].waveform = y event.dl0.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int) event.r1.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int) # Test default calibrator = CameraCalibrator( subarray=subarray, image_extractor=FullWaveformSum(subarray=subarray)) calibrator(event) np.testing.assert_allclose(event.dl1.tel[telid].image, y.sum(1), rtol=1e-4) event.calibration.tel[telid].dl1.pedestal_offset = pedestal event.calibration.tel[telid].dl1.absolute_factor = absolute event.calibration.tel[telid].dl1.relative_factor = relative # Test without timing corrections calibrator(event) dl1 = event.dl1.tel[telid] np.testing.assert_allclose(dl1.image, 1, rtol=1e-5) expected_peak_time = (mid + time_offset) / sampling_rate np.testing.assert_allclose(dl1.peak_time, expected_peak_time, rtol=1e-5) # test with timing corrections event.calibration.tel[telid].dl1.time_shift = time_offset / sampling_rate calibrator(event) # more rtol since shifting might lead to reduced integral np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-5) np.testing.assert_allclose(event.dl1.tel[telid].peak_time, mid / sampling_rate, atol=1) # test not applying time shifts # now we should be back to the result without setting time shift calibrator.apply_peak_time_shift = False calibrator.apply_waveform_time_shift = False calibrator(event) np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-4) np.testing.assert_allclose(event.dl1.tel[telid].peak_time, expected_peak_time, atol=1) # We now use GlobalPeakWindowSum to see the effect of missing charge # due to not correcting time offsets. calibrator = CameraCalibrator( subarray=subarray, image_extractor=GlobalPeakWindowSum(subarray=subarray)) calibrator(event) # test with timing corrections, should work # higher rtol because we cannot shift perfectly np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=0.01) np.testing.assert_allclose(event.dl1.tel[telid].peak_time, mid / sampling_rate, atol=1) # test deactivating timing corrections calibrator.apply_waveform_time_shift = False calibrator(event) # make sure we chose an example where the time shifts matter # charges should be quite off due to summing around global shift assert not np.allclose(event.dl1.tel[telid].image, 1, rtol=0.1) assert not np.allclose( event.dl1.tel[telid].peak_time, mid / sampling_rate, atol=1)