def test_subarray_description(): pos = {} tel = {} foclen = 16 * u.m pix_x = np.arange(1764, dtype=np.float) * u.m pix_y = np.arange(1764, dtype=np.float) * u.m for ii in range(10): tel[ii] = TelescopeDescription.guess(pix_x, pix_y, foclen) pos[ii] = (np.random.uniform(200, size=2)-100) * u.m sub = SubarrayDescription("test array", tel_positions=pos, tel_descriptions=tel) sub.info() assert sub.num_tels == 10 assert sub.tel[0].camera is not None assert len(sub.to_table()) == 10 subsub = sub.select_subarray("newsub", [1,2,3,4]) assert subsub.num_tels == 4 assert set(subsub.tels.keys()) == {1,2,3,4}
def _build_subarray_info(self, file): """ constructs a SubarrayDescription object from the info in an EventIO/HESSSIO file Parameters ---------- file: HessioFile The open pyhessio file Returns ------- SubarrayDescription : instrumental information """ telescope_ids = list(file.get_telescope_ids()) subarray = SubarrayDescription("MonteCarloArray") for tel_id in telescope_ids: try: tel = self._build_telescope_description(file, tel_id) tel_pos = u.Quantity(file.get_telescope_position(tel_id), u.m) subarray.tels[tel_id] = tel subarray.positions[tel_id] = tel_pos except self.pyhessio.HessioGeneralError: pass return subarray
def _generator(self): # container for LST data self.data = LSTDataContainer() self.data.meta['input_url'] = self.input_url self.data.meta['max_events'] = self.max_events # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile() # Instrument information for tel_id in self.data.lst.tels_with_data: assert (tel_id == 0) # only LST1 for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("LST") optics.tel_subtype = '' # to correct bug in reading # camera info from LSTCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position taken from MC from the moment tel_pos = {tel_id: [50., 50., 16] * u.m} subarray = SubarrayDescription("LST1 subarray") subarray.tels = tels subarray.positions = tel_pos self.data.inst.subarray = subarray # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific LST event data self.fill_lst_event_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def test_subarray_description(): pos = {} tel = {} n_tels = 10 for tel_id in range(1, n_tels + 1): tel[tel_id] = TelescopeDescription.from_name( optics_name="MST", camera_name="NectarCam", ) pos[tel_id] = np.random.uniform(-100, 100, size=3) * u.m sub = SubarrayDescription( "test array", tel_positions=pos, tel_descriptions=tel ) assert len(sub.telescope_types) == 1 assert str(sub) == "test array" assert sub.num_tels == n_tels assert len(sub.tel_ids) == n_tels assert sub.tel_ids[0] == 1 assert sub.tel[1].camera is not None assert 0 not in sub.tel # check that there is no tel 0 (1 is first above) assert len(sub.to_table()) == n_tels assert len(sub.camera_types) == 1 # only 1 camera type assert sub.camera_types[0] == 'NectarCam' assert sub.optics_types[0].equivalent_focal_length.to_value(u.m) == 16.0 assert sub.telescope_types[0] == 'MST:NectarCam' assert sub.tel_coords assert isinstance(sub.tel_coords, SkyCoord) assert len(sub.tel_coords) == n_tels subsub = sub.select_subarray("newsub", [2, 3, 4, 6]) assert subsub.num_tels == 4 assert set(subsub.tels.keys()) == {2, 3, 4, 6} assert subsub.tel_indices[6] == 3 assert subsub.tel_ids[3] == 6 assert len(sub.to_table(kind='optics')) == 1
def _build_subarray_info(self, file): """ constructs a SubarrayDescription object from the info in an EventIO/HESSSIO file Parameters ---------- file: HessioFile The open pyhessio file Returns ------- SubarrayDescription : instrumental information """ telescope_ids = list(file.get_telescope_ids()) subarray = SubarrayDescription("MonteCarloArray") for tel_id in telescope_ids: try: pix_pos = file.get_pixel_position(tel_id) * u.m foclen = file.get_optical_foclen(tel_id) * u.m mirror_area = file.get_mirror_area(tel_id) * u.m ** 2 num_tiles = file.get_mirror_number(tel_id) tel_pos = file.get_telescope_position(tel_id) * u.m tel = TelescopeDescription.guess(*pix_pos, equivalent_focal_length=foclen) tel.optics.mirror_area = mirror_area tel.optics.num_mirror_tiles = num_tiles subarray.tels[tel_id] = tel subarray.positions[tel_id] = tel_pos except self.pyhessio.HessioGeneralError: pass return subarray
def example_subarray(n_tels=10): """ generate a simple subarray for testing purposes """ rng = np.random.default_rng(0) pos = {} tel = {} for tel_id in range(1, n_tels + 1): tel[tel_id] = TelescopeDescription.from_name( optics_name="MST", camera_name="NectarCam" ) pos[tel_id] = rng.uniform(-100, 100, size=3) * u.m return SubarrayDescription("test array", tel_positions=pos, tel_descriptions=tel)
def subarray(): lst = TelescopeDescription.from_name("LST", "LSTCam") tels = [lst] * 4 positions = { 1: [0, 0, 0] * u.m, 2: [50, 0, 0] * u.m, 3: [0, 50, 0] * u.m, 4: [50, 50, 0] * u.m, } descriptions = {i: t for i, t in enumerate(tels, start=1)} return SubarrayDescription("test", positions, descriptions)
def create_subarray(self, tel_id=1): """ Obtain the subarray from the EventSource Returns ------- ctapipe.instrument.SubarrayDescription """ # camera info from LSTCam-[geometry_version].camgeom.fits.gz file camera_geom = load_camera_geometry(version=self.geometry_version) # get info on the camera readout: daq_time_per_sample, pulse_shape_time_step, pulse_shapes = read_pulse_shapes() camera_readout = CameraReadout('LSTCam', 1./daq_time_per_sample * u.GHz, pulse_shapes, pulse_shape_time_step, ) camera = CameraDescription('LSTCam', camera_geom, camera_readout) lst_tel_descr = TelescopeDescription( name='LST', tel_type='LST', optics=OPTICS, camera=camera ) tel_descriptions = {tel_id: lst_tel_descr} # LSTs telescope position taken from MC from the moment tel_positions = {tel_id: [50., 50., 16] * u.m} subarray = SubarrayDescription("LST1 subarray") subarray.tel_descriptions = tel_descriptions subarray.tel_positions = tel_positions subarray.tel[tel_id] = lst_tel_descr return subarray
def test_hdf_duplicate_string_repr(tmp_path): """Test writing and reading of a subarray with two telescopes that are different but have the same name. """ # test with a subarray that has two different telescopes with the same # camera tel1 = TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam") # second telescope is almost the same and as the same str repr tel2 = deepcopy(tel1) # e.g. one mirror fell off tel2.optics.num_mirror_tiles = tel1.optics.num_mirror_tiles - 1 array = SubarrayDescription( "test array", tel_positions={ 1: [0, 0, 0] * u.m, 2: [50, 0, 0] * u.m }, tel_descriptions={ 1: tel1, 2: tel2 }, ) # defensive checks to make sure we are actually testing this assert len(array.telescope_types) == 2 assert str(tel1) == str(tel2) assert tel1 != tel2 path = tmp_path / "subarray.h5" array.to_hdf(path) read = SubarrayDescription.from_hdf(path) assert array == read assert (read.tel[1].optics.num_mirror_tiles == read.tel[2].optics.num_mirror_tiles + 1)
def test_pedestal_calculator(): """ test of PedestalIntegrator """ from lstchain.calib.camera.pedestals import PedestalIntegrator tel_id = 0 n_events = 10 n_gain = 2 n_pixels = 1855 ped_level = 300 subarray = SubarrayDescription( "test array", tel_positions={0: np.zeros(3) * u.m}, tel_descriptions={ 0: TelescopeDescription.from_name( optics_name="SST-ASTRI", camera_name="CHEC" ), }, ) subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2)) subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(1, u.ns) config = Config({ "FixedWindowSum": { "apply_integration_correction": False, } }) ped_calculator = PedestalIntegrator(charge_product="FixedWindowSum", config=config, sample_size=n_events, tel_id=tel_id, subarray=subarray) # create one event data = ArrayEventContainer() data.meta['origin'] = 'test' # fill the values necessary for the pedestal calculation data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros( (n_gain, n_pixels), dtype=bool ) data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level) data.trigger.time = Time(0, format='mjd', scale='tai') while ped_calculator.num_events_seen < n_events: if ped_calculator.calculate_pedestals(data): assert data.mon.tel[tel_id].pedestal assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == ( ped_calculator.extractor.window_width.tel[tel_id] * ped_level ) assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def _generator(self): # container for NectarCAM data self.data = NectarCAMDataContainer() self.data.meta['input_url'] = self.input_url # fill data from the CameraConfig table self.fill_nectarcam_service_container_from_zfile() # Instrument information for tel_id in self.data.nectarcam.tels_with_data: assert (tel_id == 0) # only one telescope for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file geometry_version = 2 camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position tel_pos = {tel_id: [0., 0., 0] * u.m} self.subarray = SubarrayDescription("MST prototype subarray") self.subarray.tels = tels self.subarray.positions = tel_pos self.data.inst.subarray = self.subarray # loop on events for count, event in enumerate(self.multi_file): self.data.count = count # fill specific NectarCAM event data self.fill_nectarcam_event_container_from_zfile(event) # fill general R0 data self.fill_r0_container_from_zfile(event) yield self.data
def test_dl1writer(tmpdir: Path): """ Check that we can write DL1 files Parameters ---------- tmpdir : temp directory fixture """ output_path = Path(tmpdir / "events.dl1.h5") source = EventSource( get_dataset_path( "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"), max_events=20, allowed_tels=[1, 2, 3, 4], ) calibrate = CameraCalibrator(subarray=source.subarray) with DL1Writer( event_source=source, output_path=output_path, write_parameters=False, write_images=True, ) as write_dl1: write_dl1.log.level = logging.DEBUG for event in source: calibrate(event) write_dl1(event) write_dl1.write_simulation_histograms(source) assert output_path.exists() # check we can get the subarray description: sub = SubarrayDescription.from_hdf(output_path) assert sub.num_tels > 0 # check a few things in the output just to make sure there is output. For a # full test of the data model, a verify tool should be created. with tables.open_file(output_path) as h5file: images = h5file.get_node("/dl1/event/telescope/images/tel_001") assert images.col("image").max() > 0.0 assert (h5file.root._v_attrs["CTA PRODUCT DATA MODEL VERSION"] # pylint: disable=protected-access == DL1_DATA_MODEL_VERSION) shower = h5file.get_node("/simulation/event/subarray/shower") assert len(shower) > 0 assert shower.col("true_alt").mean() > 0.0 assert (shower._v_attrs["true_alt_UNIT"] == "deg") # pylint: disable=protected-access
def subarray_lst(): telid = 1 subarray = SubarrayDescription( "test array lst", tel_positions={1: np.zeros(3) * u.m, 2: np.ones(3) * u.m}, tel_descriptions={ 1: TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam"), 2: TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam"), }, ) n_pixels = subarray.tel[telid].camera.geometry.n_pixels n_samples = 30 selected_gain_channel = np.zeros(n_pixels, dtype=np.int) return subarray, telid, selected_gain_channel, n_pixels, n_samples
def test_hdf(example_subarray): import tables with tempfile.NamedTemporaryFile(suffix=".hdf5") as f: example_subarray.to_hdf(f.name) read = SubarrayDescription.from_hdf(f.name) assert example_subarray == read # test we can write the read subarray read.to_hdf(f.name, overwrite=True) for tel_id, tel in read.tel.items(): assert (tel.camera.geometry.frame.focal_length == tel.optics.equivalent_focal_length) # test if transforming works tel.camera.geometry.transform_to(TelescopeFrame()) # test that subarrays without name (v0.8.0) work: with tables.open_file(f.name, "r+") as hdf: del hdf.root.configuration.instrument.subarray._v_attrs.name no_name = SubarrayDescription.from_hdf(f.name) assert no_name.name == "Unknown" # test with a subarray that has two different telescopes with the same # camera tel = { 1: TelescopeDescription.from_name(optics_name="SST-ASTRI", camera_name="CHEC"), 2: TelescopeDescription.from_name(optics_name="SST-GCT", camera_name="CHEC"), } pos = {1: [0, 0, 0] * u.m, 2: [50, 0, 0] * u.m} array = SubarrayDescription("test array", tel_positions=pos, tel_descriptions=tel) with tempfile.NamedTemporaryFile(suffix=".hdf5") as f: array.to_hdf(f.name) read = SubarrayDescription.from_hdf(f.name) assert array == read
def read_subarray_description(filename, subarray_name='LST-1'): """ Read subarray description from an HDF5 DL1 file Parameters ---------- filename: str Returns ------- `ctapipe.instrument.subarray.SubarrayDescription` """ tel_pos = read_telescopes_positions(filename) tel_descrp = read_telescopes_descriptions(filename) return SubarrayDescription(subarray_name, tel_positions=tel_pos, tel_descriptions=tel_descrp)
def test_sw_pulse_lst(): """ Test function of sliding window extractor for LST camera pulse shape with the correction for the integration window completeness """ # prepare array with 1 LST subarray = SubarrayDescription( "LST1", tel_positions={1: np.zeros(3) * u.m}, tel_descriptions={ 1: TelescopeDescription.from_name(optics_name="LST", camera_name="LSTCam") }, ) telid = list(subarray.tel.keys())[0] n_pixels = subarray.tel[telid].camera.geometry.n_pixels n_samples = 40 readout = subarray.tel[telid].camera.readout random = np.random.RandomState(1) min_charge = 100 max_charge = 1000 charge_true = random.uniform(min_charge, max_charge, n_pixels) time_true = random.uniform(n_samples // 2 - 1, n_samples // 2 + 1, n_pixels) / readout.sampling_rate.to_value( u.GHz) waveform_model = WaveformModel.from_camera_readout(readout) waveform = waveform_model.get_waveform(charge_true, time_true, n_samples) selected_gain_channel = np.zeros(charge_true.size, dtype=np.int8) # define extractor config = Config({"SlidingWindowMaxSum": {"window_width": 8}}) extractor = SlidingWindowMaxSum(subarray=subarray) extractor = ImageExtractor.from_name("SlidingWindowMaxSum", subarray=subarray, config=config) dl1: DL1CameraContainer = extractor(waveform, telid, selected_gain_channel) print(dl1.image / charge_true) assert_allclose(dl1.image, charge_true, rtol=0.02) assert dl1.is_valid
def generate_subarray_description(dataset: DataFrame, event_id: str) -> SubarrayDescription: tel_descriptions = dict() tel_positions = dict() event_group: DataFrame = dataset.groupby("event_unique_id").get_group( event_id) for idx, tel in event_group.iterrows(): tel_id = tel["telescope_id"] tel_name = tel['type'] optics_name, camera_name = split_tel_type(tel_name) assert tel_id not in tel_descriptions tel_descriptions[tel_id] = get_telescope_description( optics_name, camera_name) tel_positions[tel_id] = [tel['x'], tel['y'], tel['z']] return SubarrayDescription(event_id, tel_positions=tel_positions, tel_descriptions=tel_descriptions)
def test_telescope_component(): from ctapipe.core import TelescopeComponent from ctapipe.instrument import SubarrayDescription, TelescopeDescription subarray = SubarrayDescription( "test", tel_positions={1: [0, 0, 0] * u.m}, tel_descriptions={1: TelescopeDescription.from_name("LST", "LSTCam")}, ) class Base(TelescopeComponent): pass class Sub(Base): pass assert isinstance(Base.from_name("Sub", subarray=subarray), Sub)
def test_tel_ids_to_mask(example_subarray): lst = TelescopeDescription.from_name("LST", "LSTCam") subarray = SubarrayDescription( "someone_counted_in_binary", tel_positions={1: [0, 0, 0] * u.m, 10: [50, 0, 0] * u.m}, tel_descriptions={1: lst, 10: lst}, ) assert np.all(subarray.tel_ids_to_mask([]) == [False, False]) assert np.all(subarray.tel_ids_to_mask([1]) == [True, False]) assert np.all(subarray.tel_ids_to_mask([10]) == [False, True]) assert np.all(subarray.tel_ids_to_mask([1, 10]) == [True, True])
def test_pedestal_calculator(): """ test of PedestalIntegrator """ tel_id = 0 n_events = 10 n_gain = 2 n_pixels = 1855 ped_level = 300 subarray = SubarrayDescription( "test array", tel_positions={0: np.zeros(3) * u.m}, tel_descriptions={ 0: TelescopeDescription.from_name( optics_name="SST-ASTRI", camera_name="CHEC" ), }, ) subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2)) subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity(1, u.ns) ped_calculator = PedestalIntegrator( subarray=subarray, charge_product="FixedWindowSum", sample_size=n_events, tel_id=tel_id, ) # create one event data = EventAndMonDataContainer() data.meta["origin"] = "test" # fill the values necessary for the pedestal calculation data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros( (n_gain, n_pixels), dtype=bool ) data.r1.tel[tel_id].waveform = np.full((2, n_pixels, 40), ped_level) data.r1.tel[tel_id].trigger_time = 1000 while ped_calculator.num_events_seen < n_events: if ped_calculator.calculate_pedestals(data): assert data.mon.tel[tel_id].pedestal assert np.mean(data.mon.tel[tel_id].pedestal.charge_median) == ( ped_calculator.extractor.window_width.tel[0] * ped_level ) assert np.mean(data.mon.tel[tel_id].pedestal.charge_std) == 0
def test_estimator_results(): """ creating some planes pointing in different directions (two north-south, two east-west) and that have a slight position errors (+- 0.1 m in one of the four cardinal directions """ horizon_frame = AltAz() p1 = SkyCoord(alt=43 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=47 * u.deg, az=45 * u.deg, frame=horizon_frame) circle1 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, 1, 0] * u.m) p1 = SkyCoord(alt=44 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=46 * u.deg, az=90 * u.deg, frame=horizon_frame) circle2 = HillasPlane(p1=p1, p2=p2, telescope_position=[1, 0, 0] * u.m) p1 = SkyCoord(alt=44.5 * u.deg, az=45 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=46.5 * u.deg, az=45 * u.deg, frame=horizon_frame) circle3 = HillasPlane(p1=p1, p2=p2, telescope_position=[0, -1, 0] * u.m) p1 = SkyCoord(alt=43.5 * u.deg, az=90 * u.deg, frame=horizon_frame) p2 = SkyCoord(alt=45.5 * u.deg, az=90 * u.deg, frame=horizon_frame) circle4 = HillasPlane(p1=p1, p2=p2, telescope_position=[-1, 0, 0] * u.m) # Create a dummy subarray # (not used here, but required to initialize the reconstructor) subarray = SubarrayDescription( "test array", tel_positions={1: np.zeros(3) * u.m}, tel_descriptions={ 1: TelescopeDescription.from_name(optics_name="SST-ASTRI", camera_name="CHEC") }, ) # creating the fit class and setting the the great circle member fit = HillasReconstructor(subarray) hillas_planes = {1: circle1, 2: circle2, 3: circle3, 4: circle4} # performing the direction fit with the minimisation algorithm # and a seed that is perpendicular to the up direction dir_fit_minimise, _ = fit.estimate_direction(hillas_planes) print("direction fit test minimise:", dir_fit_minimise)
def test_scts(): from ctapipe.instrument import TelescopeDescription, SubarrayDescription from ctapipe.image.muon.intensity_fitter import MuonIntensityFitter telescope = TelescopeDescription.from_name('SST-ASTRI', 'CHEC') subarray = SubarrayDescription( 'ssts', {0: [0, 0, 0] * u.m}, {0: telescope}, ) fitter = MuonIntensityFitter(subarray=subarray) with pytest.raises(NotImplementedError): fitter( tel_id=0, center_x=0 * u.deg, center_y=2 * u.deg, radius=1.3 * u.deg, image=np.zeros(telescope.camera.geometry.n_pixels), pedestal=np.zeros(telescope.camera.geometry.n_pixels) )
def test_subarray_description(): pos = {} tel = {} n_tels = 10 for tel_id in range(1, n_tels + 1): tel[tel_id] = TelescopeDescription.from_name( optics_name="MST", camera_name="NectarCam", ) pos[tel_id] = np.random.uniform(-100, 100, size=3) * u.m sub = SubarrayDescription( "test array", tel_positions=pos, tel_descriptions=tel ) sub.peek() assert len(sub.telescope_types) == 1 assert str(sub) == "test array" assert sub.num_tels == n_tels assert len(sub.tel_ids) == n_tels assert sub.tel_ids[0] == 1 assert sub.tel[1].camera is not None assert 0 not in sub.tel # check that there is no tel 0 (1 is first above) assert len(sub.to_table()) == n_tels assert len(sub.camera_types) == 1 # only 1 camera type assert sub.camera_types[0] == 'NectarCam' assert sub.optics_types[0].equivalent_focal_length.to_value(u.m) == 16.0 assert sub.telescope_types[0] == 'MST:NectarCam' assert sub.tel_coords assert isinstance(sub.tel_coords, SkyCoord) assert len(sub.tel_coords) == n_tels subsub = sub.select_subarray("newsub", [2, 3, 4, 6]) assert subsub.num_tels == 4 assert set(subsub.tels.keys()) == {2, 3, 4, 6} assert subsub.tel_indices[6] == 3 assert subsub.tel_ids[3] == 6 assert len(sub.to_table(kind='optics')) == 1
def prepare_subarray_info(telescope_descriptions, header): """ Constructs a SubarrayDescription object from the ``telescope_descriptions`` given by ``SimTelFile`` Parameters ---------- telescope_descriptions: dict telescope descriptions as given by ``SimTelFile.telescope_descriptions`` header: dict header as returned by ``SimTelFile.header`` Returns ------- SubarrayDescription : instrumental information """ tel_descriptions = {} # tel_id : TelescopeDescription tel_positions = {} # tel_id : TelescopeDescription for tel_id, telescope_description in telescope_descriptions.items(): cam_settings = telescope_description['camera_settings'] tel_description = TelescopeDescription.guess( cam_settings['pixel_x'] * u.m, cam_settings['pixel_y'] * u.m, equivalent_focal_length=cam_settings['focal_length'] * u.m) tel_description.optics.mirror_area = (cam_settings['mirror_area'] * u.m**2) tel_description.optics.num_mirror_tiles = ( cam_settings['mirror_area']) tel_descriptions[tel_id] = tel_description tel_idx = np.where(header['tel_id'] == tel_id)[0][0] tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m return SubarrayDescription( "MonteCarloArray", tel_positions=tel_positions, tel_descriptions=tel_descriptions, )
def test_allowed_tels(tmp_path, dl1_file, dl1_proton_file): from ctapipe.tools.dl1_merge import MergeTool from ctapipe.instrument import SubarrayDescription # create file to test 'allowed-tels' option output = tmp_path / "merged_allowed_tels.dl1.h5" ret = run_tool( MergeTool(), argv=[ str(dl1_file), str(dl1_proton_file), f"--output={output}", "--allowed-tels=[1,2]", "--overwrite", ], cwd=tmp_path, ) assert ret == 0 s = SubarrayDescription.from_hdf(output) assert s.tel.keys() == {1, 2}
def main(): dl1_filename = os.path.abspath(args.input_file) config = get_standard_config() if args.config_file is not None: try: config = read_configuration_file(os.path.abspath(args.config_file)) except ("Custom configuration could not be loaded !!!"): pass dl1_params = pd.read_hdf(dl1_filename, key=dl1_params_lstcam_key) subarray_info = SubarrayDescription.from_hdf(dl1_filename) tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1 focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length src_dep_df = pd.concat(get_source_dependent_parameters( dl1_params, config, focal_length=focal_length), axis=1) write_dataframe(src_dep_df, dl1_filename, dl1_params_src_dep_lstcam_key)
def prepare_subarray_info(self, tel_id=0): """ Constructs a SubarrayDescription object. Parameters ---------- tel_id: int Telescope identifier. Returns ------- SubarrayDescription : instrumental information """ tel_descriptions = {} # tel_id : TelescopeDescription tel_positions = {} # tel_id : TelescopeDescription # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file camera = CameraGeometry.from_name("NectarCam", self.geometry_version) tel_descr = TelescopeDescription(name='MST', tel_type='NectarCam', optics=optics, camera=camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels = tel_descr.camera.n_pixels # MST telescope position tel_positions[tel_id] = [0., 0., 0] * u.m tel_descriptions[tel_id] = tel_descr return SubarrayDescription( "Adlershof", tel_positions=tel_positions, tel_descriptions=tel_descriptions, )
def test_dl1writer_no_events(tmpdir: Path): """ Check that we can write DL1 files even when no events are given Parameters ---------- tmpdir : temp directory fixture """ output_path = Path(tmpdir / "no_events.dl1.h5") dataset = "lst_prod3_calibration_and_mcphotons.simtel.zst" with EventSource(get_dataset_path(dataset), focal_length_choice='nominal') as source: # exhaust source for _ in source: pass assert source.file_.histograms is not None with DataWriter( event_source=source, output_path=output_path, write_parameters=True, write_images=True, ) as writer: writer.log.level = logging.DEBUG writer.write_simulation_histograms(source) assert output_path.exists() # check we can get the subarray description: sub = SubarrayDescription.from_hdf(output_path) assert sub == source.subarray with tables.open_file(output_path) as h5file: assert h5file.get_node("/configuration/simulation/run") is not None assert h5file.get_node( "/simulation/service/shower_distribution") is not None
def test_image_cleaner(method): """ Test that we can construct and use a component-based ImageCleaner""" config = Config({ "TailcutsImageCleaner": { "boundary_threshold_pe": 5.0, "picture_threshold_pe": 10.0, }, "MARSImageCleaner": { "boundary_threshold_pe": 5.0, "picture_threshold_pe": 10.0, }, "FACTImageCleaner": { "boundary_threshold_pe": 5.0, "picture_threshold_pe": 10.0, "time_limit_ns": 6.0, }, }) tel = TelescopeDescription.from_name("MST", "NectarCam") subarray = SubarrayDescription(name="test", tel_positions={1: None}, tel_descriptions={1: tel}) clean = ImageCleaner.from_name(method, config=config, subarray=subarray) image = np.zeros_like(tel.camera.geometry.pix_x.value, dtype=np.float) image[10:30] = 20.0 image[31:40] = 8.0 times = np.linspace(-5, 10, image.shape[0]) mask = clean(tel_id=1, image=image, arrival_times=times) # we're not testing the algorithm here, just that it does something (for the # algorithm tests, see test_cleaning.py assert np.count_nonzero(mask) > 0
def test_flasherflatfieldcalculator(): """test of flasherFlatFieldCalculator""" tel_id = 0 n_gain = 2 n_events = 10 n_pixels = 1855 ff_level = 10000 subarray = SubarrayDescription( "test array", tel_positions={0: np.zeros(3) * u.m}, tel_descriptions={ 0: TelescopeDescription.from_name(optics_name="SST-ASTRI", camera_name="CHEC"), }, ) subarray.tel[0].camera.readout.reference_pulse_shape = np.ones((1, 2)) subarray.tel[0].camera.readout.reference_pulse_sample_width = u.Quantity( 1, u.ns) config = Config({ "FixedWindowSum": { "window_shift": 5, "window_width": 10, "peak_index": 20, "apply_integration_correction": False, } }) ff_calculator = FlasherFlatFieldCalculator(subarray=subarray, charge_product="FixedWindowSum", sample_size=n_events, tel_id=tel_id, config=config) # create one event data = ArrayEventContainer() data.meta['origin'] = 'test' data.trigger.time = Time(0, format='mjd', scale='tai') # initialize mon and r1 data data.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.zeros( (n_gain, n_pixels), dtype=bool) data.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = np.zeros( (n_gain, n_pixels), dtype=bool) data.mon.tel[tel_id].pixel_status.flatfield_failing_pixels = np.zeros( (n_gain, n_pixels), dtype=bool) data.r1.tel[tel_id].waveform = np.zeros((n_gain, n_pixels, 40), dtype=np.float32) # data.r1.tel[tel_id].trigger_time = 1000 # flat-field signal put == delta function of height ff_level at sample 20 data.r1.tel[tel_id].waveform[:, :, 20] = ff_level print(data.r1.tel[tel_id].waveform[0, 0, 20]) # First test: good event while ff_calculator.num_events_seen < n_events: if ff_calculator.calculate_relative_gain(data): assert data.mon.tel[tel_id].flatfield print(data.mon.tel[tel_id].flatfield) assert np.mean( data.mon.tel[tel_id].flatfield.charge_median) == ff_level assert np.mean( data.mon.tel[tel_id].flatfield.relative_gain_median) == 1 assert np.mean( data.mon.tel[tel_id].flatfield.relative_gain_std) == 0 # Second test: introduce some failing pixels failing_pixels_id = np.array([10, 20, 30, 40]) data.r1.tel[tel_id].waveform[:, failing_pixels_id, :] = 0 data.mon.tel[ tel_id].pixel_status.pedestal_failing_pixels[:, failing_pixels_id] = True while ff_calculator.num_events_seen < n_events: if ff_calculator.calculate_relative_gain(data): # working pixel have good gain assert ( data.mon.tel[tel_id].flatfield.relative_gain_median[0, 0] == 1) # bad pixels do non influence the gain assert np.mean( data.mon.tel[tel_id].flatfield.relative_gain_std) == 0
def __init__(self, **kwargs): """ Constructor Parameters ---------- kwargs: dict Parameters to be passed. NOTE: The file mask of the data to read can be passed with the 'input_url' parameter. """ try: import uproot except ImportError: raise ImportError( "The 'uproot' package is required for the DLMAGICEventSource class." ) self.file_list = glob.glob(kwargs['input_url']) self.file_list.sort() # Since EventSource can not handle file wild cards as input_url # We substitute the input_url with first file matching # the specified file mask. del kwargs['input_url'] super().__init__(input_url=self.file_list[0], **kwargs) # get run number mask = r".*_za\d+to\d+_\d_(\d+)_([A-Z]+)_.*" parsed_info = re.findall(mask, self.file_list[0]) self.run_number = parsed_info[0][0] # MAGIC telescope positions in m wrt. to the center of CTA simulations self.magic_tel_positions = { 1: [-27.24, -146.66, 50.00] * u.m, 2: [-96.44, -96.77, 51.00] * u.m } self.magic_tel_positions = self.magic_tel_positions # MAGIC telescope description optics = OpticsDescription.from_name('MAGIC') geom = CameraGeometry.from_name('MAGICCam') # Camera Readout for NectarCam used as a placeholder readout = CameraReadout( 'MAGICCam', sampling_rate=u.Quantity(1, u.GHz), reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]), reference_pulse_sample_width=u.Quantity(1, u.ns)) camera = CameraDescription('MAGICCam', geom, readout) self.magic_tel_description = TelescopeDescription(name='MAGIC', tel_type='LST', optics=optics, camera=camera) self.magic_tel_descriptions = { 1: self.magic_tel_description, 2: self.magic_tel_description } self.magic_subarray = SubarrayDescription('MAGIC', self.magic_tel_positions, self.magic_tel_descriptions) # Open ROOT files self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = None, None, None, None, None for file in self.file_list: uproot_file = uproot.open(file) if "_Y_" in file: if "_M1_" in file: self.calib_M1 = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] elif "_M2_" in file: self.calib_M2 = uproot_file["Events"] if "_I_" in file: if "_M1_" in file: self.star_M1 = uproot_file["Events"] elif "_M2_" in file: self.star_M2 = uproot_file["Events"] if "_S_" in file: self.superstar = uproot_file["Events"] self.meta = uproot_file["RunHeaders"] self._mc_header = self._parse_mc_header()
def main(): args = parser.parse_args() custom_config = {} if args.config_file is not None: try: custom_config = read_configuration_file( os.path.abspath(args.config_file)) except ("Custom configuration could not be loaded !!!"): pass config = replace_config(standard_config, custom_config) data = pd.read_hdf(args.input_file, key=dl1_params_lstcam_key) if 'lh_fit_config' in config.keys(): lhfit_data = pd.read_hdf(args.input_file, key=dl1_likelihood_params_lstcam_key) if np.all(lhfit_data['obs_id'] == data['obs_id']) & np.all( lhfit_data['event_id'] == data['event_id']): lhfit_data.drop({'obs_id', 'event_id'}, axis=1, inplace=True) lhfit_keys = lhfit_data.keys() data = pd.concat([data, lhfit_data], axis=1) # if real data, add deltat t to dataframe keys data = add_delta_t_key(data) # Dealing with pointing missing values. This happened when `ucts_time` was invalid. if 'alt_tel' in data.columns and 'az_tel' in data.columns \ and (np.isnan(data.alt_tel).any() or np.isnan(data.az_tel).any()): # make sure there is a least one good pointing value to interp from. if np.isfinite(data.alt_tel).any() and np.isfinite(data.az_tel).any(): data = impute_pointing(data) else: data.alt_tel = -np.pi / 2. data.az_tel = -np.pi / 2. # Get trained RF path for reconstruction: file_reg_energy = os.path.join(args.path_models, 'reg_energy.sav') file_cls_gh = os.path.join(args.path_models, 'cls_gh.sav') if config['disp_method'] == 'disp_vector': file_disp_vector = os.path.join(args.path_models, 'reg_disp_vector.sav') elif config['disp_method'] == 'disp_norm_sign': file_disp_norm = os.path.join(args.path_models, 'reg_disp_norm.sav') file_disp_sign = os.path.join(args.path_models, 'cls_disp_sign.sav') subarray_info = SubarrayDescription.from_hdf(args.input_file) tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1 focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length # Apply the models to the data # Source-independent analysis if not config['source_dependent']: data = filter_events( data, filters=config["events_filters"], finite_params=config['energy_regression_features'] + config['disp_regression_features'] + config['particle_classification_features'] + config['disp_classification_features'], ) if config['disp_method'] == 'disp_vector': dl2 = dl1_to_dl2.apply_models(data, file_cls_gh, file_reg_energy, reg_disp_vector=file_disp_vector, focal_length=focal_length, custom_config=config) elif config['disp_method'] == 'disp_norm_sign': dl2 = dl1_to_dl2.apply_models(data, file_cls_gh, file_reg_energy, reg_disp_norm=file_disp_norm, cls_disp_sign=file_disp_sign, focal_length=focal_length, custom_config=config) # Source-dependent analysis if config['source_dependent']: # if source-dependent parameters are already in dl1 data, just read those data. if dl1_params_src_dep_lstcam_key in get_dataset_keys(args.input_file): data_srcdep = get_srcdep_params(args.input_file) # if not, source-dependent parameters are added now else: data_srcdep = pd.concat(dl1_to_dl2.get_source_dependent_parameters( data, config, focal_length=focal_length), axis=1) dl2_srcdep_dict = {} srcindep_keys = data.keys() srcdep_assumed_positions = data_srcdep.columns.levels[0] for i, k in enumerate(srcdep_assumed_positions): data_with_srcdep_param = pd.concat([data, data_srcdep[k]], axis=1) data_with_srcdep_param = filter_events( data_with_srcdep_param, filters=config["events_filters"], finite_params=config['energy_regression_features'] + config['disp_regression_features'] + config['particle_classification_features'] + config['disp_classification_features'], ) if config['disp_method'] == 'disp_vector': dl2_df = dl1_to_dl2.apply_models( data_with_srcdep_param, file_cls_gh, file_reg_energy, reg_disp_vector=file_disp_vector, focal_length=focal_length, custom_config=config) elif config['disp_method'] == 'disp_norm_sign': dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param, file_cls_gh, file_reg_energy, reg_disp_norm=file_disp_norm, cls_disp_sign=file_disp_sign, focal_length=focal_length, custom_config=config) dl2_srcdep = dl2_df.drop(srcindep_keys, axis=1) dl2_srcdep_dict[k] = dl2_srcdep if i == 0: dl2_srcindep = dl2_df[srcindep_keys] os.makedirs(args.output_dir, exist_ok=True) output_file = os.path.join( args.output_dir, os.path.basename(args.input_file).replace('dl1', 'dl2', 1)) if os.path.exists(output_file): raise IOError(output_file + ' exists, exiting.') dl1_keys = get_dataset_keys(args.input_file) if dl1_images_lstcam_key in dl1_keys: dl1_keys.remove(dl1_images_lstcam_key) if dl1_params_lstcam_key in dl1_keys: dl1_keys.remove(dl1_params_lstcam_key) if dl1_params_src_dep_lstcam_key in dl1_keys: dl1_keys.remove(dl1_params_src_dep_lstcam_key) if dl1_likelihood_params_lstcam_key in dl1_keys: dl1_keys.remove(dl1_likelihood_params_lstcam_key) metadata = global_metadata() write_metadata(metadata, output_file) with open_file(args.input_file, 'r') as h5in: with open_file(output_file, 'a') as h5out: # Write the selected DL1 info for k in dl1_keys: if not k.startswith('/'): k = '/' + k path, name = k.rsplit('/', 1) if path not in h5out: grouppath, groupname = path.rsplit('/', 1) g = h5out.create_group(grouppath, groupname, createparents=True) else: g = h5out.get_node(path) h5in.copy_node(k, g, overwrite=True) # need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata if not config['source_dependent']: if 'lh_fit_config' not in config.keys(): write_dl2_dataframe(dl2, output_file, config=config, meta=metadata) else: dl2_onlylhfit = dl2[lhfit_keys] dl2.drop(lhfit_keys, axis=1, inplace=True) write_dl2_dataframe(dl2, output_file, config=config, meta=metadata) write_dataframe(dl2_onlylhfit, output_file, dl2_likelihood_params_lstcam_key, config=config, meta=metadata) else: write_dl2_dataframe(dl2_srcindep, output_file, config=config, meta=metadata) write_dataframe(pd.concat(dl2_srcdep_dict, axis=1), output_file, dl2_params_src_dep_lstcam_key, config=config, meta=metadata)
def test_dl1writer_int(tmpdir: Path): """ Check that we can write DL1 files Parameters ---------- tmpdir : temp directory fixture """ output_path = Path(tmpdir / "events.dl1.h5") source = EventSource( get_dataset_path( "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"), max_events=20, allowed_tels=[1, 2, 3, 4], ) calibrate = CameraCalibrator(subarray=source.subarray) events = [] with DL1Writer( event_source=source, output_path=output_path, write_parameters=False, write_images=True, transform_image=True, image_dtype="int32", image_scale=10, transform_peak_time=True, peak_time_dtype="int16", peak_time_scale=100, ) as write_dl1: write_dl1.log.level = logging.DEBUG for event in source: calibrate(event) write_dl1(event) events.append(deepcopy(event)) write_dl1.write_simulation_histograms(source) assert output_path.exists() # check we can get the subarray description: sub = SubarrayDescription.from_hdf(output_path) assert sub.num_tels > 0 # check a few things in the output just to make sure there is output. For a # full test of the data model, a verify tool should be created. with tables.open_file(output_path) as h5file: images = h5file.get_node("/dl1/event/telescope/images/tel_001") assert len(images) > 0 assert images.col("image").dtype == np.int32 assert images.col("peak_time").dtype == np.int16 assert images.col("image").max() > 0.0 # make sure it is readable by the event source and matches the images for event in EventSource(output_path): for tel_id, dl1 in event.dl1.tel.items(): original_image = events[event.count].dl1.tel[tel_id].image read_image = dl1.image assert np.allclose(original_image, read_image, atol=0.1) original_peaktime = events[event.count].dl1.tel[tel_id].peak_time read_peaktime = dl1.peak_time assert np.allclose(original_peaktime, read_peaktime, atol=0.01)
def prepare_subarray_info(telescope_descriptions, header): """ Constructs a SubarrayDescription object from the ``telescope_descriptions`` given by ``SimTelFile`` Parameters ---------- telescope_descriptions: dict telescope descriptions as given by ``SimTelFile.telescope_descriptions`` header: dict header as returned by ``SimTelFile.header`` Returns ------- SubarrayDescription : instrumental information """ tel_descriptions = {} # tel_id : TelescopeDescription tel_positions = {} # tel_id : TelescopeDescription for tel_id, telescope_description in telescope_descriptions.items(): cam_settings = telescope_description['camera_settings'] n_pixels = cam_settings['n_pixels'] focal_length = u.Quantity(cam_settings['focal_length'], u.m) try: telescope = guess_telescope(n_pixels, focal_length) except ValueError: telescope = UNKNOWN_TELESCOPE pixel_shape = cam_settings['pixel_shape'][0] try: pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type( pixel_shape) except ValueError: warnings.warn( f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}', UnknownPixelShapeWarning, ) pix_type = 'hexagon' pix_rotation = '0d' camera = CameraGeometry( telescope.camera_name, pix_id=np.arange(n_pixels), pix_x=u.Quantity(cam_settings['pixel_x'], u.m), pix_y=u.Quantity(cam_settings['pixel_y'], u.m), pix_area=u.Quantity(cam_settings['pixel_area'], u.m**2), pix_type=pix_type, pix_rotation=pix_rotation, cam_rotation=-Angle(cam_settings['cam_rot'], u.rad), apply_derotation=True, ) optics = OpticsDescription( name=telescope.name, num_mirrors=cam_settings['n_mirrors'], equivalent_focal_length=focal_length, mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2), num_mirror_tiles=cam_settings['n_mirrors'], ) tel_descriptions[tel_id] = TelescopeDescription( name=telescope.name, type=telescope.type, camera=camera, optics=optics, ) tel_idx = np.where(header['tel_id'] == tel_id)[0][0] tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m return SubarrayDescription( "MonteCarloArray", tel_positions=tel_positions, tel_descriptions=tel_descriptions, )