def test_subarray_description():
pos = {}
tel = {}
foclen = 16 * u.m
pix_x = np.arange(1764, dtype=np.float) * u.m
pix_y = np.arange(1764, dtype=np.float) * u.m
for ii in range(10):
tel[ii] = TelescopeDescription.guess(pix_x, pix_y, foclen)
pos[ii] = (np.random.uniform(200, size=2)-100) * u.m
sub = SubarrayDescription("test array",
tel_positions=pos,
tel_descriptions=tel)
sub.info()
assert sub.num_tels == 10
assert sub.tel[0].camera is not None
assert len(sub.to_table()) == 10
subsub = sub.select_subarray("newsub", [1,2,3,4])
assert subsub.num_tels == 4
assert set(subsub.tels.keys()) == {1,2,3,4}
def _build_subarray_info(self, file):
"""
constructs a SubarrayDescription object from the info in an
EventIO/HESSSIO file
Parameters
----------
file: HessioFile
The open pyhessio file
Returns
-------
SubarrayDescription :
instrumental information
"""
telescope_ids = list(file.get_telescope_ids())
subarray = SubarrayDescription("MonteCarloArray")
for tel_id in telescope_ids:
try:
tel = self._build_telescope_description(file, tel_id)
tel_pos = u.Quantity(file.get_telescope_position(tel_id), u.m)
subarray.tels[tel_id] = tel
subarray.positions[tel_id] = tel_pos
except self.pyhessio.HessioGeneralError:
pass
return subarray
def _generator(self):
# container for LST data
self.data = LSTDataContainer()
self.data.meta['input_url'] = self.input_url
self.data.meta['max_events'] = self.max_events
# fill LST data from the CameraConfig table
self.fill_lst_service_container_from_zfile()
# Instrument information
for tel_id in self.data.lst.tels_with_data:
assert (tel_id == 0) # only LST1 for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("LST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("LSTCam", geometry_version)
tel_descr = TelescopeDescription(optics, camera)
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position taken from MC from the moment
tel_pos = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tels = tels
subarray.positions = tel_pos
self.data.inst.subarray = subarray
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific LST event data
self.fill_lst_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def test_subarray_description():
pos = {}
tel = {}
n_tels = 10
for tel_id in range(1, n_tels + 1):
tel[tel_id] = TelescopeDescription.from_name(
optics_name="MST",
camera_name="NectarCam",
)
pos[tel_id] = np.random.uniform(-100, 100, size=3) * u.m
sub = SubarrayDescription(
"test array",
tel_positions=pos,
tel_descriptions=tel
)
assert len(sub.telescope_types) == 1
assert str(sub) == "test array"
assert sub.num_tels == n_tels
assert len(sub.tel_ids) == n_tels
assert sub.tel_ids[0] == 1
assert sub.tel[1].camera is not None
assert 0 not in sub.tel # check that there is no tel 0 (1 is first above)
assert len(sub.to_table()) == n_tels
assert len(sub.camera_types) == 1 # only 1 camera type
assert sub.camera_types[0] == 'NectarCam'
assert sub.optics_types[0].equivalent_focal_length.to_value(u.m) == 16.0
assert sub.telescope_types[0] == 'MST:NectarCam'
assert sub.tel_coords
assert isinstance(sub.tel_coords, SkyCoord)
assert len(sub.tel_coords) == n_tels
subsub = sub.select_subarray("newsub", [2, 3, 4, 6])
assert subsub.num_tels == 4
assert set(subsub.tels.keys()) == {2, 3, 4, 6}
assert subsub.tel_indices[6] == 3
assert subsub.tel_ids[3] == 6
assert len(sub.to_table(kind='optics')) == 1
def _build_subarray_info(self, file):
"""
constructs a SubarrayDescription object from the info in an
EventIO/HESSSIO file
Parameters
----------
file: HessioFile
The open pyhessio file
Returns
-------
SubarrayDescription :
instrumental information
"""
telescope_ids = list(file.get_telescope_ids())
subarray = SubarrayDescription("MonteCarloArray")
for tel_id in telescope_ids:
try:
pix_pos = file.get_pixel_position(tel_id) * u.m
foclen = file.get_optical_foclen(tel_id) * u.m
mirror_area = file.get_mirror_area(tel_id) * u.m ** 2
num_tiles = file.get_mirror_number(tel_id)
tel_pos = file.get_telescope_position(tel_id) * u.m
tel = TelescopeDescription.guess(*pix_pos,
equivalent_focal_length=foclen)
tel.optics.mirror_area = mirror_area
tel.optics.num_mirror_tiles = num_tiles
subarray.tels[tel_id] = tel
subarray.positions[tel_id] = tel_pos
except self.pyhessio.HessioGeneralError:
pass
return subarray
def example_subarray(n_tels=10):
""" generate a simple subarray for testing purposes """
rng = np.random.default_rng(0)
pos = {}
tel = {}
for tel_id in range(1, n_tels + 1):
tel[tel_id] = TelescopeDescription.from_name(
optics_name="MST", camera_name="NectarCam"
)
pos[tel_id] = rng.uniform(-100, 100, size=3) * u.m
return SubarrayDescription("test array", tel_positions=pos, tel_descriptions=tel)
def subarray():
lst = TelescopeDescription.from_name("LST", "LSTCam")
tels = [lst] * 4
positions = {
1: [0, 0, 0] * u.m,
2: [50, 0, 0] * u.m,
3: [0, 50, 0] * u.m,
4: [50, 50, 0] * u.m,
}
descriptions = {i: t for i, t in enumerate(tels, start=1)}
return SubarrayDescription("test", positions, descriptions)
def create_subarray(self, tel_id=1):
"""
Obtain the subarray from the EventSource
Returns
-------
ctapipe.instrument.SubarrayDescription
"""
# camera info from LSTCam-[geometry_version].camgeom.fits.gz file
camera_geom = load_camera_geometry(version=self.geometry_version)
# get info on the camera readout:
daq_time_per_sample, pulse_shape_time_step, pulse_shapes = read_pulse_shapes()
camera_readout = CameraReadout('LSTCam',
1./daq_time_per_sample * u.GHz,
pulse_shapes,
pulse_shape_time_step,
)
camera = CameraDescription('LSTCam', camera_geom, camera_readout)
lst_tel_descr = TelescopeDescription(
name='LST', tel_type='LST', optics=OPTICS, camera=camera
)
tel_descriptions = {tel_id: lst_tel_descr}
# LSTs telescope position taken from MC from the moment
tel_positions = {tel_id: [50., 50., 16] * u.m}
subarray = SubarrayDescription("LST1 subarray")
subarray.tel_descriptions = tel_descriptions
subarray.tel_positions = tel_positions
subarray.tel[tel_id] = lst_tel_descr
return subarray
def test_hdf_duplicate_string_repr(tmp_path):
"""Test writing and reading of a subarray with two telescopes that
are different but have the same name.
"""
# test with a subarray that has two different telescopes with the same
# camera
tel1 = TelescopeDescription.from_name(optics_name="LST",
camera_name="LSTCam")
# second telescope is almost the same and as the same str repr
tel2 = deepcopy(tel1)
# e.g. one mirror fell off
tel2.optics.num_mirror_tiles = tel1.optics.num_mirror_tiles - 1
array = SubarrayDescription(
"test array",
tel_positions={
1: [0, 0, 0] * u.m,
2: [50, 0, 0] * u.m
},
tel_descriptions={
1: tel1,
2: tel2
},
)
# defensive checks to make sure we are actually testing this
assert len(array.telescope_types) == 2
assert str(tel1) == str(tel2)
assert tel1 != tel2
path = tmp_path / "subarray.h5"
array.to_hdf(path)
read = SubarrayDescription.from_hdf(path)
assert array == read
assert (read.tel[1].optics.num_mirror_tiles ==
read.tel[2].optics.num_mirror_tiles + 1)
def test_pedestal_calculator():
""" test of PedestalIntegrator """
from lstchain.calib.camera.pedestals import PedestalIntegrator
tel_id = 0
n_events = 10
n_gain = 2
n_pixels = 1855
ped_level = 300
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0: TelescopeDescription.from_name(
optics_name="SST-ASTRI", camera_name="CHEC"
),
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(1, u.ns)
config = Config({
"FixedWindowSum": {
"apply_integration_correction": False,
}
})
ped_calculator = PedestalIntegrator(charge_product="FixedWindowSum",
config=config,
sample_size=n_events,
tel_id=tel_id,
subarray=subarray)
# create one event
data = ArrayEventContainer()
data.meta['origin'] = 'test'
# fill the values necessary for the pedestal calculation
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool
)
data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level)
data.trigger.time = Time(0, format='mjd', scale='tai')
while ped_calculator.num_events_seen < n_events:
if ped_calculator.calculate_pedestals(data):
assert data.mon.tel[tel_id].pedestal
assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == (
ped_calculator.extractor.window_width.tel[tel_id] * ped_level
)
assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def _generator(self):
# container for NectarCAM data
self.data = NectarCAMDataContainer()
self.data.meta['input_url'] = self.input_url
# fill data from the CameraConfig table
self.fill_nectarcam_service_container_from_zfile()
# Instrument information
for tel_id in self.data.nectarcam.tels_with_data:
assert (tel_id == 0) # only one telescope for the moment (id = 0)
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
geometry_version = 2
camera = CameraGeometry.from_name("NectarCam", geometry_version)
tel_descr = TelescopeDescription(optics, camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
tels = {tel_id: tel_descr}
# LSTs telescope position
tel_pos = {tel_id: [0., 0., 0] * u.m}
self.subarray = SubarrayDescription("MST prototype subarray")
self.subarray.tels = tels
self.subarray.positions = tel_pos
self.data.inst.subarray = self.subarray
# loop on events
for count, event in enumerate(self.multi_file):
self.data.count = count
# fill specific NectarCAM event data
self.fill_nectarcam_event_container_from_zfile(event)
# fill general R0 data
self.fill_r0_container_from_zfile(event)
yield self.data
def test_dl1writer(tmpdir: Path):
"""
Check that we can write DL1 files
Parameters
----------
tmpdir :
temp directory fixture
"""
output_path = Path(tmpdir / "events.dl1.h5")
source = EventSource(
get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"),
max_events=20,
allowed_tels=[1, 2, 3, 4],
)
calibrate = CameraCalibrator(subarray=source.subarray)
with DL1Writer(
event_source=source,
output_path=output_path,
write_parameters=False,
write_images=True,
) as write_dl1:
write_dl1.log.level = logging.DEBUG
for event in source:
calibrate(event)
write_dl1(event)
write_dl1.write_simulation_histograms(source)
assert output_path.exists()
# check we can get the subarray description:
sub = SubarrayDescription.from_hdf(output_path)
assert sub.num_tels > 0
# check a few things in the output just to make sure there is output. For a
# full test of the data model, a verify tool should be created.
with tables.open_file(output_path) as h5file:
images = h5file.get_node("/dl1/event/telescope/images/tel_001")
assert images.col("image").max() > 0.0
assert (h5file.root._v_attrs["CTA PRODUCT DATA MODEL VERSION"] # pylint: disable=protected-access
== DL1_DATA_MODEL_VERSION)
shower = h5file.get_node("/simulation/event/subarray/shower")
assert len(shower) > 0
assert shower.col("true_alt").mean() > 0.0
assert (shower._v_attrs["true_alt_UNIT"] == "deg") # pylint: disable=protected-access
def subarray_lst():
telid = 1
subarray = SubarrayDescription(
"test array lst",
tel_positions={1: np.zeros(3) * u.m, 2: np.ones(3) * u.m},
tel_descriptions={
1: TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam"),
2: TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam"),
},
)
n_pixels = subarray.tel[telid].camera.geometry.n_pixels
n_samples = 30
selected_gain_channel = np.zeros(n_pixels, dtype=np.int)
return subarray, telid, selected_gain_channel, n_pixels, n_samples
def test_hdf(example_subarray):
import tables
with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
example_subarray.to_hdf(f.name)
read = SubarrayDescription.from_hdf(f.name)
assert example_subarray == read
# test we can write the read subarray
read.to_hdf(f.name, overwrite=True)
for tel_id, tel in read.tel.items():
assert (tel.camera.geometry.frame.focal_length ==
tel.optics.equivalent_focal_length)
# test if transforming works
tel.camera.geometry.transform_to(TelescopeFrame())
# test that subarrays without name (v0.8.0) work:
with tables.open_file(f.name, "r+") as hdf:
del hdf.root.configuration.instrument.subarray._v_attrs.name
no_name = SubarrayDescription.from_hdf(f.name)
assert no_name.name == "Unknown"
# test with a subarray that has two different telescopes with the same
# camera
tel = {
1:
TelescopeDescription.from_name(optics_name="SST-ASTRI",
camera_name="CHEC"),
2:
TelescopeDescription.from_name(optics_name="SST-GCT",
camera_name="CHEC"),
}
pos = {1: [0, 0, 0] * u.m, 2: [50, 0, 0] * u.m}
array = SubarrayDescription("test array",
tel_positions=pos,
tel_descriptions=tel)
with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
array.to_hdf(f.name)
read = SubarrayDescription.from_hdf(f.name)
assert array == read
def read_subarray_description(filename, subarray_name='LST-1'):
"""
Read subarray description from an HDF5 DL1 file
Parameters
----------
filename: str
Returns
-------
`ctapipe.instrument.subarray.SubarrayDescription`
"""
tel_pos = read_telescopes_positions(filename)
tel_descrp = read_telescopes_descriptions(filename)
return SubarrayDescription(subarray_name,
tel_positions=tel_pos,
tel_descriptions=tel_descrp)
def test_sw_pulse_lst():
"""
Test function of sliding window extractor for LST camera pulse shape with
the correction for the integration window completeness
"""
# prepare array with 1 LST
subarray = SubarrayDescription(
"LST1",
tel_positions={1: np.zeros(3) * u.m},
tel_descriptions={
1:
TelescopeDescription.from_name(optics_name="LST",
camera_name="LSTCam")
},
)
telid = list(subarray.tel.keys())[0]
n_pixels = subarray.tel[telid].camera.geometry.n_pixels
n_samples = 40
readout = subarray.tel[telid].camera.readout
random = np.random.RandomState(1)
min_charge = 100
max_charge = 1000
charge_true = random.uniform(min_charge, max_charge, n_pixels)
time_true = random.uniform(n_samples // 2 - 1, n_samples // 2 + 1,
n_pixels) / readout.sampling_rate.to_value(
u.GHz)
waveform_model = WaveformModel.from_camera_readout(readout)
waveform = waveform_model.get_waveform(charge_true, time_true, n_samples)
selected_gain_channel = np.zeros(charge_true.size, dtype=np.int8)
# define extractor
config = Config({"SlidingWindowMaxSum": {"window_width": 8}})
extractor = SlidingWindowMaxSum(subarray=subarray)
extractor = ImageExtractor.from_name("SlidingWindowMaxSum",
subarray=subarray,
config=config)
dl1: DL1CameraContainer = extractor(waveform, telid, selected_gain_channel)
print(dl1.image / charge_true)
assert_allclose(dl1.image, charge_true, rtol=0.02)
assert dl1.is_valid
def generate_subarray_description(dataset: DataFrame,
event_id: str) -> SubarrayDescription:
tel_descriptions = dict()
tel_positions = dict()
event_group: DataFrame = dataset.groupby("event_unique_id").get_group(
event_id)
for idx, tel in event_group.iterrows():
tel_id = tel["telescope_id"]
tel_name = tel['type']
optics_name, camera_name = split_tel_type(tel_name)
assert tel_id not in tel_descriptions
tel_descriptions[tel_id] = get_telescope_description(
optics_name, camera_name)
tel_positions[tel_id] = [tel['x'], tel['y'], tel['z']]
return SubarrayDescription(event_id,
tel_positions=tel_positions,
tel_descriptions=tel_descriptions)
def test_telescope_component():
from ctapipe.core import TelescopeComponent
from ctapipe.instrument import SubarrayDescription, TelescopeDescription
subarray = SubarrayDescription(
"test",
tel_positions={1: [0, 0, 0] * u.m},
tel_descriptions={1: TelescopeDescription.from_name("LST", "LSTCam")},
)
class Base(TelescopeComponent):
pass
class Sub(Base):
pass
assert isinstance(Base.from_name("Sub", subarray=subarray), Sub)
def test_tel_ids_to_mask(example_subarray):
lst = TelescopeDescription.from_name("LST", "LSTCam")
subarray = SubarrayDescription(
"someone_counted_in_binary",
tel_positions={1: [0, 0, 0] * u.m, 10: [50, 0, 0] * u.m},
tel_descriptions={1: lst, 10: lst},
)
assert np.all(subarray.tel_ids_to_mask([]) == [False, False])
assert np.all(subarray.tel_ids_to_mask([1]) == [True, False])
assert np.all(subarray.tel_ids_to_mask([10]) == [False, True])
assert np.all(subarray.tel_ids_to_mask([1, 10]) == [True, True])
def test_pedestal_calculator():
""" test of PedestalIntegrator """
tel_id = 0
n_events = 10
n_gain = 2
n_pixels = 1855
ped_level = 300
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0: TelescopeDescription.from_name(
optics_name="SST-ASTRI", camera_name="CHEC"
),
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(1, u.ns)
ped_calculator = PedestalIntegrator(
subarray=subarray,
charge_product="FixedWindowSum",
sample_size=n_events,
tel_id=tel_id,
)
# create one event
data = EventAndMonDataContainer()
data.meta["origin"] = "test"
# fill the values necessary for the pedestal calculation
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool
)
data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level)
data.r1.tel[tel_id].trigger_time = 1000
while ped_calculator.num_events_seen < n_events:
if ped_calculator.calculate_pedestals(data):
assert data.mon.tel[tel_id].pedestal
assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == (
ped_calculator.extractor.window_width.tel[0] * ped_level
)
assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def test_estimator_results():
"""
creating some planes pointing in different directions (two
north-south, two east-west) and that have a slight position errors (+-
0.1 m in one of the four cardinal directions """
horizon_frame = AltAz()
p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m)
p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m)
p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame=horizon_frame)
circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m)
p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame=horizon_frame)
p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame=horizon_frame)
circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m)
# Create a dummy subarray
# (not used here, but required to initialize the reconstructor)
subarray = SubarrayDescription(
"test array",
tel_positions={1: np.zeros(3) * u.m},
tel_descriptions={
1:
TelescopeDescription.from_name(optics_name="SST-ASTRI",
camera_name="CHEC")
},
)
# creating the fit class and setting the the great circle member
fit = HillasReconstructor(subarray)
hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4}
# performing the direction fit with the minimisation algorithm
# and a seed that is perpendicular to the up direction
dir_fit_minimise, _ = fit.estimate_direction(hillas_planes)
print("direction fit test minimise:", dir_fit_minimise)
def test_scts():
from ctapipe.instrument import TelescopeDescription, SubarrayDescription
from ctapipe.image.muon.intensity_fitter import MuonIntensityFitter
telescope = TelescopeDescription.from_name('SST-ASTRI', 'CHEC')
subarray = SubarrayDescription(
'ssts', {0: [0, 0, 0] * u.m}, {0: telescope},
)
fitter = MuonIntensityFitter(subarray=subarray)
with pytest.raises(NotImplementedError):
fitter(
tel_id=0,
center_x=0 * u.deg,
center_y=2 * u.deg,
radius=1.3 * u.deg,
image=np.zeros(telescope.camera.geometry.n_pixels),
pedestal=np.zeros(telescope.camera.geometry.n_pixels)
)
def test_subarray_description():
pos = {}
tel = {}
n_tels = 10
for tel_id in range(1, n_tels + 1):
tel[tel_id] = TelescopeDescription.from_name(
optics_name="MST",
camera_name="NectarCam",
)
pos[tel_id] = np.random.uniform(-100, 100, size=3) * u.m
sub = SubarrayDescription(
"test array",
tel_positions=pos,
tel_descriptions=tel
)
sub.peek()
assert len(sub.telescope_types) == 1
assert str(sub) == "test array"
assert sub.num_tels == n_tels
assert len(sub.tel_ids) == n_tels
assert sub.tel_ids[0] == 1
assert sub.tel[1].camera is not None
assert 0 not in sub.tel # check that there is no tel 0 (1 is first above)
assert len(sub.to_table()) == n_tels
assert len(sub.camera_types) == 1 # only 1 camera type
assert sub.camera_types[0] == 'NectarCam'
assert sub.optics_types[0].equivalent_focal_length.to_value(u.m) == 16.0
assert sub.telescope_types[0] == 'MST:NectarCam'
assert sub.tel_coords
assert isinstance(sub.tel_coords, SkyCoord)
assert len(sub.tel_coords) == n_tels
subsub = sub.select_subarray("newsub", [2, 3, 4, 6])
assert subsub.num_tels == 4
assert set(subsub.tels.keys()) == {2, 3, 4, 6}
assert subsub.tel_indices[6] == 3
assert subsub.tel_ids[3] == 6
assert len(sub.to_table(kind='optics')) == 1
def prepare_subarray_info(telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description['camera_settings']
tel_description = TelescopeDescription.guess(
cam_settings['pixel_x'] * u.m,
cam_settings['pixel_y'] * u.m,
equivalent_focal_length=cam_settings['focal_length'] * u.m)
tel_description.optics.mirror_area = (cam_settings['mirror_area'] *
u.m**2)
tel_description.optics.num_mirror_tiles = (
cam_settings['mirror_area'])
tel_descriptions[tel_id] = tel_description
tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
def test_allowed_tels(tmp_path, dl1_file, dl1_proton_file):
from ctapipe.tools.dl1_merge import MergeTool
from ctapipe.instrument import SubarrayDescription
# create file to test 'allowed-tels' option
output = tmp_path / "merged_allowed_tels.dl1.h5"
ret = run_tool(
MergeTool(),
argv=[
str(dl1_file),
str(dl1_proton_file),
f"--output={output}",
"--allowed-tels=[1,2]",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0
s = SubarrayDescription.from_hdf(output)
assert s.tel.keys() == {1, 2}
def main():
dl1_filename = os.path.abspath(args.input_file)
config = get_standard_config()
if args.config_file is not None:
try:
config = read_configuration_file(os.path.abspath(args.config_file))
except ("Custom configuration could not be loaded !!!"):
pass
dl1_params = pd.read_hdf(dl1_filename, key=dl1_params_lstcam_key)
subarray_info = SubarrayDescription.from_hdf(dl1_filename)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length
src_dep_df = pd.concat(get_source_dependent_parameters(
dl1_params, config, focal_length=focal_length),
axis=1)
write_dataframe(src_dep_df, dl1_filename, dl1_params_src_dep_lstcam_key)
def prepare_subarray_info(self, tel_id=0):
"""
Constructs a SubarrayDescription object.
Parameters
----------
tel_id: int
Telescope identifier.
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
# optics info from standard optics.fits.gz file
optics = OpticsDescription.from_name("MST")
optics.tel_subtype = '' # to correct bug in reading
# camera info from NectarCam-[geometry_version].camgeom.fits.gz file
camera = CameraGeometry.from_name("NectarCam", self.geometry_version)
tel_descr = TelescopeDescription(name='MST',
tel_type='NectarCam',
optics=optics,
camera=camera)
tel_descr.optics.tel_subtype = '' # to correct bug in reading
self.n_camera_pixels = tel_descr.camera.n_pixels
# MST telescope position
tel_positions[tel_id] = [0., 0., 0] * u.m
tel_descriptions[tel_id] = tel_descr
return SubarrayDescription(
"Adlershof",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
def test_dl1writer_no_events(tmpdir: Path):
"""
Check that we can write DL1 files even when no events are given
Parameters
----------
tmpdir :
temp directory fixture
"""
output_path = Path(tmpdir / "no_events.dl1.h5")
dataset = "lst_prod3_calibration_and_mcphotons.simtel.zst"
with EventSource(get_dataset_path(dataset),
focal_length_choice='nominal') as source:
# exhaust source
for _ in source:
pass
assert source.file_.histograms is not None
with DataWriter(
event_source=source,
output_path=output_path,
write_parameters=True,
write_images=True,
) as writer:
writer.log.level = logging.DEBUG
writer.write_simulation_histograms(source)
assert output_path.exists()
# check we can get the subarray description:
sub = SubarrayDescription.from_hdf(output_path)
assert sub == source.subarray
with tables.open_file(output_path) as h5file:
assert h5file.get_node("/configuration/simulation/run") is not None
assert h5file.get_node(
"/simulation/service/shower_distribution") is not None
def test_image_cleaner(method):
""" Test that we can construct and use a component-based ImageCleaner"""
config = Config({
"TailcutsImageCleaner": {
"boundary_threshold_pe": 5.0,
"picture_threshold_pe": 10.0,
},
"MARSImageCleaner": {
"boundary_threshold_pe": 5.0,
"picture_threshold_pe": 10.0,
},
"FACTImageCleaner": {
"boundary_threshold_pe": 5.0,
"picture_threshold_pe": 10.0,
"time_limit_ns": 6.0,
},
})
tel = TelescopeDescription.from_name("MST", "NectarCam")
subarray = SubarrayDescription(name="test",
tel_positions={1: None},
tel_descriptions={1: tel})
clean = ImageCleaner.from_name(method, config=config, subarray=subarray)
image = np.zeros_like(tel.camera.geometry.pix_x.value, dtype=np.float)
image[10:30] = 20.0
image[31:40] = 8.0
times = np.linspace(-5, 10, image.shape[0])
mask = clean(tel_id=1, image=image, arrival_times=times)
# we're not testing the algorithm here, just that it does something (for the
# algorithm tests, see test_cleaning.py
assert np.count_nonzero(mask) > 0
def test_flasherflatfieldcalculator():
"""test of flasherFlatFieldCalculator"""
tel_id = 0
n_gain = 2
n_events = 10
n_pixels = 1855
ff_level = 10000
subarray = SubarrayDescription(
"test array",
tel_positions={0: np.zeros(3) * u.m},
tel_descriptions={
0:
TelescopeDescription.from_name(optics_name="SST-ASTRI",
camera_name="CHEC"),
},
)
subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2))
subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(
1, u.ns)
config = Config({
"FixedWindowSum": {
"window_shift": 5,
"window_width": 10,
"peak_index": 20,
"apply_integration_correction": False,
}
})
ff_calculator = FlasherFlatFieldCalculator(subarray=subarray,
charge_product="FixedWindowSum",
sample_size=n_events,
tel_id=tel_id,
config=config)
# create one event
data = ArrayEventContainer()
data.meta['origin'] = 'test'
data.trigger.time = Time(0, format='mjd', scale='tai')
# initialize mon and r1 data
data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.mon.tel[tel_id].pixel_status.flatfield_failing_pixels = np.zeros(
(n_gain, n_pixels), dtype=bool)
data.r1.tel[tel_id].waveform = np.zeros((n_gain, n_pixels, 40),
dtype=np.float32)
# data.r1.tel[tel_id].trigger_time = 1000
# flat-field signal put == delta function of height ff_level at sample 20
data.r1.tel[tel_id].waveform[:, :, 20] = ff_level
print(data.r1.tel[tel_id].waveform[0, 0, 20])
# First test: good event
while ff_calculator.num_events_seen < n_events:
if ff_calculator.calculate_relative_gain(data):
assert data.mon.tel[tel_id].flatfield
print(data.mon.tel[tel_id].flatfield)
assert np.mean(
data.mon.tel[tel_id].flatfield.charge_median) == ff_level
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_median) == 1
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_std) == 0
# Second test: introduce some failing pixels
failing_pixels_id = np.array([10, 20, 30, 40])
data.r1.tel[tel_id].waveform[:, failing_pixels_id, :] = 0
data.mon.tel[
tel_id].pixel_status.pedestal_failing_pixels[:,
failing_pixels_id] = True
while ff_calculator.num_events_seen < n_events:
if ff_calculator.calculate_relative_gain(data):
# working pixel have good gain
assert (
data.mon.tel[tel_id].flatfield.relative_gain_median[0, 0] == 1)
# bad pixels do non influence the gain
assert np.mean(
data.mon.tel[tel_id].flatfield.relative_gain_std) == 0
def __init__(self, **kwargs):
"""
Constructor
Parameters
----------
kwargs: dict
Parameters to be passed.
NOTE: The file mask of the data to read can be passed with
the 'input_url' parameter.
"""
try:
import uproot
except ImportError:
raise ImportError(
"The 'uproot' package is required for the DLMAGICEventSource class."
)
self.file_list = glob.glob(kwargs['input_url'])
self.file_list.sort()
# Since EventSource can not handle file wild cards as input_url
# We substitute the input_url with first file matching
# the specified file mask.
del kwargs['input_url']
super().__init__(input_url=self.file_list[0], **kwargs)
# get run number
mask = r".*_za\d+to\d+_\d_(\d+)_([A-Z]+)_.*"
parsed_info = re.findall(mask, self.file_list[0])
self.run_number = parsed_info[0][0]
# MAGIC telescope positions in m wrt. to the center of CTA simulations
self.magic_tel_positions = {
1: [-27.24, -146.66, 50.00] * u.m,
2: [-96.44, -96.77, 51.00] * u.m
}
self.magic_tel_positions = self.magic_tel_positions
# MAGIC telescope description
optics = OpticsDescription.from_name('MAGIC')
geom = CameraGeometry.from_name('MAGICCam')
# Camera Readout for NectarCam used as a placeholder
readout = CameraReadout(
'MAGICCam',
sampling_rate=u.Quantity(1, u.GHz),
reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]),
reference_pulse_sample_width=u.Quantity(1, u.ns))
camera = CameraDescription('MAGICCam', geom, readout)
self.magic_tel_description = TelescopeDescription(name='MAGIC',
tel_type='LST',
optics=optics,
camera=camera)
self.magic_tel_descriptions = {
1: self.magic_tel_description,
2: self.magic_tel_description
}
self.magic_subarray = SubarrayDescription('MAGIC',
self.magic_tel_positions,
self.magic_tel_descriptions)
# Open ROOT files
self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = None, None, None, None, None
for file in self.file_list:
uproot_file = uproot.open(file)
if "_Y_" in file:
if "_M1_" in file:
self.calib_M1 = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
elif "_M2_" in file:
self.calib_M2 = uproot_file["Events"]
if "_I_" in file:
if "_M1_" in file:
self.star_M1 = uproot_file["Events"]
elif "_M2_" in file:
self.star_M2 = uproot_file["Events"]
if "_S_" in file:
self.superstar = uproot_file["Events"]
self.meta = uproot_file["RunHeaders"]
self._mc_header = self._parse_mc_header()
def main():
args = parser.parse_args()
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(
os.path.abspath(args.config_file))
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
data = pd.read_hdf(args.input_file, key=dl1_params_lstcam_key)
if 'lh_fit_config' in config.keys():
lhfit_data = pd.read_hdf(args.input_file,
key=dl1_likelihood_params_lstcam_key)
if np.all(lhfit_data['obs_id'] == data['obs_id']) & np.all(
lhfit_data['event_id'] == data['event_id']):
lhfit_data.drop({'obs_id', 'event_id'}, axis=1, inplace=True)
lhfit_keys = lhfit_data.keys()
data = pd.concat([data, lhfit_data], axis=1)
# if real data, add deltat t to dataframe keys
data = add_delta_t_key(data)
# Dealing with pointing missing values. This happened when `ucts_time` was invalid.
if 'alt_tel' in data.columns and 'az_tel' in data.columns \
and (np.isnan(data.alt_tel).any() or np.isnan(data.az_tel).any()):
# make sure there is a least one good pointing value to interp from.
if np.isfinite(data.alt_tel).any() and np.isfinite(data.az_tel).any():
data = impute_pointing(data)
else:
data.alt_tel = -np.pi / 2.
data.az_tel = -np.pi / 2.
# Get trained RF path for reconstruction:
file_reg_energy = os.path.join(args.path_models, 'reg_energy.sav')
file_cls_gh = os.path.join(args.path_models, 'cls_gh.sav')
if config['disp_method'] == 'disp_vector':
file_disp_vector = os.path.join(args.path_models,
'reg_disp_vector.sav')
elif config['disp_method'] == 'disp_norm_sign':
file_disp_norm = os.path.join(args.path_models, 'reg_disp_norm.sav')
file_disp_sign = os.path.join(args.path_models, 'cls_disp_sign.sav')
subarray_info = SubarrayDescription.from_hdf(args.input_file)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length
# Apply the models to the data
# Source-independent analysis
if not config['source_dependent']:
data = filter_events(
data,
filters=config["events_filters"],
finite_params=config['energy_regression_features'] +
config['disp_regression_features'] +
config['particle_classification_features'] +
config['disp_classification_features'],
)
if config['disp_method'] == 'disp_vector':
dl2 = dl1_to_dl2.apply_models(data,
file_cls_gh,
file_reg_energy,
reg_disp_vector=file_disp_vector,
focal_length=focal_length,
custom_config=config)
elif config['disp_method'] == 'disp_norm_sign':
dl2 = dl1_to_dl2.apply_models(data,
file_cls_gh,
file_reg_energy,
reg_disp_norm=file_disp_norm,
cls_disp_sign=file_disp_sign,
focal_length=focal_length,
custom_config=config)
# Source-dependent analysis
if config['source_dependent']:
# if source-dependent parameters are already in dl1 data, just read those data.
if dl1_params_src_dep_lstcam_key in get_dataset_keys(args.input_file):
data_srcdep = get_srcdep_params(args.input_file)
# if not, source-dependent parameters are added now
else:
data_srcdep = pd.concat(dl1_to_dl2.get_source_dependent_parameters(
data, config, focal_length=focal_length),
axis=1)
dl2_srcdep_dict = {}
srcindep_keys = data.keys()
srcdep_assumed_positions = data_srcdep.columns.levels[0]
for i, k in enumerate(srcdep_assumed_positions):
data_with_srcdep_param = pd.concat([data, data_srcdep[k]], axis=1)
data_with_srcdep_param = filter_events(
data_with_srcdep_param,
filters=config["events_filters"],
finite_params=config['energy_regression_features'] +
config['disp_regression_features'] +
config['particle_classification_features'] +
config['disp_classification_features'],
)
if config['disp_method'] == 'disp_vector':
dl2_df = dl1_to_dl2.apply_models(
data_with_srcdep_param,
file_cls_gh,
file_reg_energy,
reg_disp_vector=file_disp_vector,
focal_length=focal_length,
custom_config=config)
elif config['disp_method'] == 'disp_norm_sign':
dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param,
file_cls_gh,
file_reg_energy,
reg_disp_norm=file_disp_norm,
cls_disp_sign=file_disp_sign,
focal_length=focal_length,
custom_config=config)
dl2_srcdep = dl2_df.drop(srcindep_keys, axis=1)
dl2_srcdep_dict[k] = dl2_srcdep
if i == 0:
dl2_srcindep = dl2_df[srcindep_keys]
os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(
args.output_dir,
os.path.basename(args.input_file).replace('dl1', 'dl2', 1))
if os.path.exists(output_file):
raise IOError(output_file + ' exists, exiting.')
dl1_keys = get_dataset_keys(args.input_file)
if dl1_images_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_images_lstcam_key)
if dl1_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_lstcam_key)
if dl1_params_src_dep_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_src_dep_lstcam_key)
if dl1_likelihood_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_likelihood_params_lstcam_key)
metadata = global_metadata()
write_metadata(metadata, output_file)
with open_file(args.input_file, 'r') as h5in:
with open_file(output_file, 'a') as h5out:
# Write the selected DL1 info
for k in dl1_keys:
if not k.startswith('/'):
k = '/' + k
path, name = k.rsplit('/', 1)
if path not in h5out:
grouppath, groupname = path.rsplit('/', 1)
g = h5out.create_group(grouppath,
groupname,
createparents=True)
else:
g = h5out.get_node(path)
h5in.copy_node(k, g, overwrite=True)
# need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata
if not config['source_dependent']:
if 'lh_fit_config' not in config.keys():
write_dl2_dataframe(dl2, output_file, config=config, meta=metadata)
else:
dl2_onlylhfit = dl2[lhfit_keys]
dl2.drop(lhfit_keys, axis=1, inplace=True)
write_dl2_dataframe(dl2, output_file, config=config, meta=metadata)
write_dataframe(dl2_onlylhfit,
output_file,
dl2_likelihood_params_lstcam_key,
config=config,
meta=metadata)
else:
write_dl2_dataframe(dl2_srcindep,
output_file,
config=config,
meta=metadata)
write_dataframe(pd.concat(dl2_srcdep_dict, axis=1),
output_file,
dl2_params_src_dep_lstcam_key,
config=config,
meta=metadata)
def test_dl1writer_int(tmpdir: Path):
"""
Check that we can write DL1 files
Parameters
----------
tmpdir :
temp directory fixture
"""
output_path = Path(tmpdir / "events.dl1.h5")
source = EventSource(
get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"),
max_events=20,
allowed_tels=[1, 2, 3, 4],
)
calibrate = CameraCalibrator(subarray=source.subarray)
events = []
with DL1Writer(
event_source=source,
output_path=output_path,
write_parameters=False,
write_images=True,
transform_image=True,
image_dtype="int32",
image_scale=10,
transform_peak_time=True,
peak_time_dtype="int16",
peak_time_scale=100,
) as write_dl1:
write_dl1.log.level = logging.DEBUG
for event in source:
calibrate(event)
write_dl1(event)
events.append(deepcopy(event))
write_dl1.write_simulation_histograms(source)
assert output_path.exists()
# check we can get the subarray description:
sub = SubarrayDescription.from_hdf(output_path)
assert sub.num_tels > 0
# check a few things in the output just to make sure there is output. For a
# full test of the data model, a verify tool should be created.
with tables.open_file(output_path) as h5file:
images = h5file.get_node("/dl1/event/telescope/images/tel_001")
assert len(images) > 0
assert images.col("image").dtype == np.int32
assert images.col("peak_time").dtype == np.int16
assert images.col("image").max() > 0.0
# make sure it is readable by the event source and matches the images
for event in EventSource(output_path):
for tel_id, dl1 in event.dl1.tel.items():
original_image = events[event.count].dl1.tel[tel_id].image
read_image = dl1.image
assert np.allclose(original_image, read_image, atol=0.1)
original_peaktime = events[event.count].dl1.tel[tel_id].peak_time
read_peaktime = dl1.peak_time
assert np.allclose(original_peaktime, read_peaktime, atol=0.01)
def prepare_subarray_info(telescope_descriptions, header):
"""
Constructs a SubarrayDescription object from the
``telescope_descriptions`` given by ``SimTelFile``
Parameters
----------
telescope_descriptions: dict
telescope descriptions as given by ``SimTelFile.telescope_descriptions``
header: dict
header as returned by ``SimTelFile.header``
Returns
-------
SubarrayDescription :
instrumental information
"""
tel_descriptions = {} # tel_id : TelescopeDescription
tel_positions = {} # tel_id : TelescopeDescription
for tel_id, telescope_description in telescope_descriptions.items():
cam_settings = telescope_description['camera_settings']
n_pixels = cam_settings['n_pixels']
focal_length = u.Quantity(cam_settings['focal_length'], u.m)
try:
telescope = guess_telescope(n_pixels, focal_length)
except ValueError:
telescope = UNKNOWN_TELESCOPE
pixel_shape = cam_settings['pixel_shape'][0]
try:
pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(
pixel_shape)
except ValueError:
warnings.warn(
f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}',
UnknownPixelShapeWarning,
)
pix_type = 'hexagon'
pix_rotation = '0d'
camera = CameraGeometry(
telescope.camera_name,
pix_id=np.arange(n_pixels),
pix_x=u.Quantity(cam_settings['pixel_x'], u.m),
pix_y=u.Quantity(cam_settings['pixel_y'], u.m),
pix_area=u.Quantity(cam_settings['pixel_area'], u.m**2),
pix_type=pix_type,
pix_rotation=pix_rotation,
cam_rotation=-Angle(cam_settings['cam_rot'], u.rad),
apply_derotation=True,
)
optics = OpticsDescription(
name=telescope.name,
num_mirrors=cam_settings['n_mirrors'],
equivalent_focal_length=focal_length,
mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2),
num_mirror_tiles=cam_settings['n_mirrors'],
)
tel_descriptions[tel_id] = TelescopeDescription(
name=telescope.name,
type=telescope.type,
camera=camera,
optics=optics,
)
tel_idx = np.where(header['tel_id'] == tel_id)[0][0]
tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m
return SubarrayDescription(
"MonteCarloArray",
tel_positions=tel_positions,
tel_descriptions=tel_descriptions,
)
