def test_pedestal_calculator():
""" test of PedestalIntegrator """
from lstchain.calib.camera.pedestals import PedestalIntegrator
tel_id = 0
n_events = 10
n_gain = 2
n_pixels = 1855
ped_level = 300
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0: TelescopeDescription.from_name(
optics_name="SST-ASTRI", camera_name="CHEC"
),
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(1, u.ns)
config = Config({
"FixedWindowSum": {
"apply_integration_correction": False,
}
})
ped_calculator = PedestalIntegrator(charge_product="FixedWindowSum",
config=config,
sample_size=n_events,
tel_id=tel_id,
subarray=subarray)
# create one event
data = ArrayEventContainer()
data.meta['origin'] = 'test'
# fill the values necessary for the pedestal calculation
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool
)
data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level)
data.trigger.time = Time(0, format='mjd', scale='tai')
while ped_calculator.num_events_seen < n_events:
if ped_calculator.calculate_pedestals(data):
assert data.mon.tel[tel_id].pedestal
assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == (
ped_calculator.extractor.window_width.tel[tel_id] * ped_level
)
assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def test_pedestal_calculator():
""" test of PedestalIntegrator """
tel_id = 0
n_events = 10
n_gain = 2
n_pixels = 1855
ped_level = 300
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0:
TelescopeDescription.from_name(optics_name="SST-ASTRI",
camera_name="CHEC")
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(
1, u.ns)
ped_calculator = PedestalIntegrator(
subarray=subarray,
charge_product="FixedWindowSum",
sample_size=n_events,
tel_id=tel_id,
)
# create one event
data = ArrayEventContainer()
data.meta["origin"] = "test"
data.trigger.time = Time.now()
# fill the values necessary for the pedestal calculation
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level)
while ped_calculator.num_events_seen < n_events:
if ped_calculator.calculate_pedestals(data):
assert data.mon.tel[tel_id].pedestal
assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == (
ped_calculator.extractor.window_width.tel[0] * ped_level)
assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def test_event_filter_config():
from ctapipe.utils import EventTypeFilter
config = Config({"EventTypeFilter": {"allowed_types": [EventType.SUBARRAY.value]}})
event_filter = EventTypeFilter(config=config)
e = ArrayEventContainer()
e.trigger.event_type = EventType.DARK_PEDESTAL
assert not event_filter(e)
e.trigger.event_type = EventType.SUBARRAY
assert event_filter(e)
def test_rescale_dl1_charge():
event = ArrayEventContainer()
tel_ids = [1, 3]
images = {}
for tel_id in tel_ids:
images[tel_id] = np.random.rand(1855)
event.dl1.tel[tel_id].image = copy(images[tel_id])
rescaling_factor = np.random.rand() * 10
rescale_dl1_charge(event, rescaling_factor)
for tel_id in tel_ids:
np.testing.assert_allclose(event.dl1.tel[tel_id].image,
images[tel_id] * rescaling_factor)
def test_event_filter():
from ctapipe.utils import EventTypeFilter
event_filter = EventTypeFilter(
allowed_types={EventType.SUBARRAY, EventType.FLATFIELD}
)
e = ArrayEventContainer()
e.trigger.event_type = EventType.SUBARRAY
assert event_filter(e)
e.trigger.event_type = EventType.FLATFIELD
assert event_filter(e)
e.trigger.event_type = EventType.DARK_PEDESTAL
assert not event_filter(e)
def test_check_dl0_empty(example_event, example_subarray):
calibrator = CameraCalibrator(subarray=example_subarray)
telid = list(example_event.r0.tel)[0]
calibrator._calibrate_dl0(example_event, telid)
waveform = example_event.dl0.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.dl0.tel[telid].waveform = None
calibrator._calibrate_dl1(example_event, telid)
assert example_event.dl1.tel[telid].image is None
assert calibrator._check_dl0_empty(None) is True
assert calibrator._check_dl0_empty(waveform) is False
calibrator = CameraCalibrator(subarray=example_subarray)
event = ArrayEventContainer()
event.dl1.tel[telid].image = np.full(2048, 2)
with pytest.warns(UserWarning):
calibrator(event)
assert (event.dl1.tel[telid].image == 2).all()
def test_check_r1_empty(example_event, example_subarray):
calibrator = CameraCalibrator(subarray=example_subarray)
telid = list(example_event.r0.tel)[0]
waveform = example_event.r1.tel[telid].waveform.copy()
with pytest.warns(UserWarning):
example_event.r1.tel[telid].waveform = None
calibrator._calibrate_dl0(example_event, telid)
assert example_event.dl0.tel[telid].waveform is None
assert calibrator._check_r1_empty(None) is True
assert calibrator._check_r1_empty(waveform) is False
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=FullWaveformSum(subarray=example_subarray),
)
event = ArrayEventContainer()
event.dl0.tel[telid].waveform = np.full((2048, 128), 2)
with pytest.warns(UserWarning):
calibrator(event)
assert (event.dl0.tel[telid].waveform == 2).all()
assert (event.dl1.tel[telid].image == 2 * 128).all()
def test_flasherflatfieldcalculator():
"""test of flasherFlatFieldCalculator"""
tel_id = 0
n_gain = 2
n_events = 10
n_pixels = 1855
ff_level = 10000
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0:
TelescopeDescription.from_name(optics_name="SST-ASTRI",
camera_name="CHEC"),
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(
1, u.ns)
config = Config({
"FixedWindowSum": {
"window_shift": 5,
"window_width": 10,
"peak_index": 20,
"apply_integration_correction": False,
}
})
ff_calculator = FlasherFlatFieldCalculator(subarray=subarray,
charge_product="FixedWindowSum",
sample_size=n_events,
tel_id=tel_id,
config=config)
# create one event
data = ArrayEventContainer()
data.meta['origin'] = 'test'
data.trigger.time = Time(0, format='mjd', scale='tai')
# initialize mon and r1 data
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.mon.tel[tel_id].pixel_status.flatfield_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.r1.tel[tel_id].waveform = np.zeros((n_gain, n_pixels, 40),
dtype=np.float32)
# data.r1.tel[tel_id].trigger_time = 1000
# flat-field signal put == delta function of height ff_level at sample 20
data.r1.tel[tel_id].waveform[:, :, 20] = ff_level
print(data.r1.tel[tel_id].waveform[0, 0, 20])
# First test: good event
while ff_calculator.num_events_seen < n_events:
if ff_calculator.calculate_relative_gain(data):
assert data.mon.tel[tel_id].flatfield
print(data.mon.tel[tel_id].flatfield)
assert np.mean(
data.mon.tel[tel_id].flatfield.charge_median) == ff_level
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_median) == 1
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_std) == 0
# Second test: introduce some failing pixels
failing_pixels_id = np.array([10, 20, 30, 40])
data.r1.tel[tel_id].waveform[:, failing_pixels_id, :] = 0
data.mon.tel[
tel_id].pixel_status.pedestal_failing_pixels[:,
failing_pixels_id] = True
while ff_calculator.num_events_seen < n_events:
if ff_calculator.calculate_relative_gain(data):
# working pixel have good gain
assert (
data.mon.tel[tel_id].flatfield.relative_gain_median[0, 0] == 1)
# bad pixels do non influence the gain
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_std) == 0
def test_dl1_charge_calib(example_subarray):
# copy because we mutate the camera, should not affect other tests
subarray = deepcopy(example_subarray)
camera = subarray.tel[1].camera
# test with a sampling_rate different than 1 to
# test if we handle time vs. slices correctly
sampling_rate = 2
camera.readout.sampling_rate = sampling_rate * u.GHz
n_pixels = camera.geometry.n_pixels
n_samples = 96
mid = n_samples // 2
pulse_sigma = 6
random = np.random.RandomState(1)
x = np.arange(n_samples)
# Randomize times and create pulses
time_offset = random.uniform(-10, +10, n_pixels)
y = norm.pdf(x, mid + time_offset[:, np.newaxis],
pulse_sigma).astype("float32")
camera.readout.reference_pulse_shape = norm.pdf(x, mid,
pulse_sigma)[np.newaxis, :]
camera.readout.reference_pulse_sample_width = 1 / camera.readout.sampling_rate
# Define absolute calibration coefficients
absolute = random.uniform(100, 1000, n_pixels).astype("float32")
y *= absolute[:, np.newaxis]
# Define relative coefficients
relative = random.normal(1, 0.01, n_pixels)
y /= relative[:, np.newaxis]
# Define pedestal
pedestal = random.uniform(-4, 4, n_pixels)
y += pedestal[:, np.newaxis]
event = ArrayEventContainer()
telid = list(subarray.tel.keys())[0]
event.dl0.tel[telid].waveform = y
event.dl0.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int)
event.r1.tel[telid].selected_gain_channel = np.zeros(len(y), dtype=int)
# Test default
calibrator = CameraCalibrator(
subarray=subarray, image_extractor=FullWaveformSum(subarray=subarray))
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, y.sum(1), rtol=1e-4)
event.calibration.tel[telid].dl1.pedestal_offset = pedestal
event.calibration.tel[telid].dl1.absolute_factor = absolute
event.calibration.tel[telid].dl1.relative_factor = relative
# Test without timing corrections
calibrator(event)
dl1 = event.dl1.tel[telid]
np.testing.assert_allclose(dl1.image, 1, rtol=1e-5)
expected_peak_time = (mid + time_offset) / sampling_rate
np.testing.assert_allclose(dl1.peak_time, expected_peak_time, rtol=1e-5)
# test with timing corrections
event.calibration.tel[telid].dl1.time_shift = time_offset / sampling_rate
calibrator(event)
# more rtol since shifting might lead to reduced integral
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-5)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
mid / sampling_rate,
atol=1)
# test not applying time shifts
# now we should be back to the result without setting time shift
calibrator.apply_peak_time_shift = False
calibrator.apply_waveform_time_shift = False
calibrator(event)
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=1e-4)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
expected_peak_time,
atol=1)
# We now use GlobalPeakWindowSum to see the effect of missing charge
# due to not correcting time offsets.
calibrator = CameraCalibrator(
subarray=subarray,
image_extractor=GlobalPeakWindowSum(subarray=subarray))
calibrator(event)
# test with timing corrections, should work
# higher rtol because we cannot shift perfectly
np.testing.assert_allclose(event.dl1.tel[telid].image, 1, rtol=0.01)
np.testing.assert_allclose(event.dl1.tel[telid].peak_time,
mid / sampling_rate,
atol=1)
# test deactivating timing corrections
calibrator.apply_waveform_time_shift = False
calibrator(event)
# make sure we chose an example where the time shifts matter
# charges should be quite off due to summing around global shift
assert not np.allclose(event.dl1.tel[telid].image, 1, rtol=0.1)
assert not np.allclose(
event.dl1.tel[telid].peak_time, mid / sampling_rate, atol=1)
def _generate_events(self):
"""
Yield ArrayEventContainer to iterate through events.
"""
data = ArrayEventContainer()
# Maybe take some other metadata, but there are still some 'unknown'
# written out by the stage1 tool
data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
if DataLevel.DL1_IMAGES in self.datalevels:
image_iterators = {
tel.name: self.file_.root.dl1.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.dl1.event.telescope.images
}
if self.has_simulated_dl1:
simulated_image_iterators = {
tel.name: self.file_.root.simulation.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.simulation.event.telescope.images
}
if DataLevel.DL1_PARAMETERS in self.datalevels:
param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/dl1/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
TimingParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
PeakTimeStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.has_simulated_dl1:
simulated_param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/simulation/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.is_simulation:
# simulated shower wide information
mc_shower_reader = HDF5TableReader(self.file_).read(
"/simulation/event/subarray/shower",
SimulatedShowerContainer(),
prefixes="true",
)
# Setup iterators for the array events
events = HDF5TableReader(self.file_).read(
"/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()]
)
array_pointing_finder = IndexFinder(
self.file_.root.dl1.monitoring.subarray.pointing.col("time")
)
tel_pointing_finder = {
tel.name: IndexFinder(tel.col("time"))
for tel in self.file_.root.dl1.monitoring.telescope.pointing
}
for counter, array_event in enumerate(events):
data.dl1.tel.clear()
data.simulation.tel.clear()
data.pointing.tel.clear()
data.trigger.tel.clear()
data.count = counter
data.trigger, data.index = next(events)
data.trigger.tels_with_trigger = (
np.where(data.trigger.tels_with_trigger)[0] + 1
) # +1 to match array index to telescope id
# Maybe there is a simpler way to do this
# Beware: tels_with_trigger contains all triggered telescopes whereas
# the telescope trigger table contains only the subset of
# allowed_tels given during the creation of the dl1 file
for i in self.file_.root.dl1.event.telescope.trigger.where(
f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
):
if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
continue
if self.datamodel_version == "v1.0.0":
data.trigger.tel[i["tel_id"]].time = i["telescopetrigger_time"]
else:
data.trigger.tel[i["tel_id"]].time = i["time"]
self._fill_array_pointing(data, array_pointing_finder)
self._fill_telescope_pointing(data, tel_pointing_finder)
if self.is_simulation:
data.simulation.shower = next(mc_shower_reader)
for tel in data.trigger.tel.keys():
if self.allowed_tels and tel not in self.allowed_tels:
continue
if self.has_simulated_dl1:
simulated = data.simulation.tel[tel]
dl1 = data.dl1.tel[tel]
if DataLevel.DL1_IMAGES in self.datalevels:
if f"tel_{tel:03d}" not in image_iterators.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the image table."
)
continue
image_row = next(image_iterators[f"tel_{tel:03d}"])
dl1.image = image_row["image"]
dl1.peak_time = image_row["peak_time"]
dl1.image_mask = image_row["image_mask"]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in simulated_image_iterators.keys():
logger.warning(
f"Triggered telescope {tel} is missing "
"from the simulated image table, but was present at the "
"reconstructed image table."
)
continue
simulated_image_row = next(
simulated_image_iterators[f"tel_{tel:03d}"]
)
simulated.true_image = simulated_image_row["true_image"]
if DataLevel.DL1_PARAMETERS in self.datalevels:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the parameters table."
)
continue
# Is there a smarter way to unpack this?
# Best would probbaly be if we could directly read
# into the ImageParametersContainer
params = next(param_readers[f"tel_{tel:03d}"])
dl1.parameters.hillas = params[0]
dl1.parameters.timing = params[1]
dl1.parameters.leakage = params[2]
dl1.parameters.concentration = params[3]
dl1.parameters.morphology = params[4]
dl1.parameters.intensity_statistics = params[5]
dl1.parameters.peak_time_statistics = params[6]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the simulated parameters table, but was "
"present at the reconstructed parameters table."
)
continue
simulated_params = next(
simulated_param_readers[f"tel_{tel:03d}"]
)
simulated.true_parameters.hillas = simulated_params[0]
simulated.true_parameters.leakage = simulated_params[1]
simulated.true_parameters.concentration = simulated_params[2]
simulated.true_parameters.morphology = simulated_params[3]
simulated.true_parameters.intensity_statistics = simulated_params[
4
]
yield data
def test_array_display():
""" check that we can do basic array display functionality """
from ctapipe.visualization.mpl_array import ArrayDisplay
from ctapipe.image import timing_parameters
from ctapipe.containers import (
ArrayEventContainer,
DL1Container,
DL1CameraContainer,
ImageParametersContainer,
CoreParametersContainer,
)
# build a test subarray:
tels = dict()
tel_pos = dict()
for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m):
tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam")
tel_pos[ii + 1] = pos
sub = SubarrayDescription(name="TestSubarray",
tel_positions=tel_pos,
tel_descriptions=tels)
# Create a fake event containing telescope-wise information about
# the image directions projected on the ground
event = ArrayEventContainer()
event.dl1 = DL1Container()
event.dl1.tel = {1: DL1CameraContainer(), 2: DL1CameraContainer()}
event.dl1.tel[1].parameters = ImageParametersContainer()
event.dl1.tel[2].parameters = ImageParametersContainer()
event.dl1.tel[2].parameters.core = CoreParametersContainer()
event.dl1.tel[1].parameters.core = CoreParametersContainer()
event.dl1.tel[1].parameters.core.psi = u.Quantity(2.0, unit=u.deg)
event.dl1.tel[2].parameters.core.psi = u.Quantity(1.0, unit=u.deg)
ad = ArrayDisplay(subarray=sub)
ad.set_vector_rho_phi(1 * u.m, 90 * u.deg)
# try setting a value
vals = np.ones(sub.num_tels)
ad.values = vals
assert (vals == ad.values).all()
# test UV field ...
# ...with colors by telescope type
ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m)
# ...with scalar color
ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m, c=3)
geom = CameraGeometry.from_name("LSTCam")
rot_angle = 20 * u.deg
hillas = CameraHillasParametersContainer(x=0 * u.m,
y=0 * u.m,
psi=rot_angle)
# test using hillas params CameraFrame:
hillas_dict = {
1: CameraHillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg),
2: CameraHillasParametersContainer(length=20000 * u.cm, psi="95deg"),
}
grad = 2
intercept = 1
timing_rot20 = timing_parameters(
geom,
image=np.ones(geom.n_pixels),
peak_time=intercept + grad * geom.pix_x.value,
hillas_parameters=hillas,
cleaning_mask=np.ones(geom.n_pixels, dtype=bool),
)
gradient_dict = {1: timing_rot20.slope.value, 2: timing_rot20.slope.value}
core_dict = {
tel_id: dl1.parameters.core.psi
for tel_id, dl1 in event.dl1.tel.items()
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
# test using hillas params for divergent pointing in telescopeframe:
hillas_dict = {
1:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=90 * u.deg),
2:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=95 * u.deg),
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
# test using hillas params for parallel pointing in telescopeframe:
hillas_dict = {
1:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=90 * u.deg),
2:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=95 * u.deg),
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# test negative time_gradients
gradient_dict = {1: -0.03, 2: -0.02}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# and very small
gradient_dict = {1: 0.003, 2: 0.002}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# Test background contour
ad.background_contour(
x=np.array([0, 1, 2]),
y=np.array([0, 1, 2]),
background=np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]]),
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
ad.add_labels()
ad.remove_labels()