Exemplo n.º 1
0
    def ctapipe_subarray(self):
        from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \
            CameraGeometry, CameraReadout, CameraDescription, OpticsDescription
        import astropy.units as u

        geom = CameraGeometry("sstcam", self.mapping.pixel.i,
                              u.Quantity(self.mapping.pixel.x, 'm'),
                              u.Quantity(self.mapping.pixel.y, 'm'),
                              u.Quantity(self.mapping.pixel.size, 'm')**2,
                              'square')

        readout = CameraReadout(
            "sstcam", u.Quantity(1 / self.waveform_sample_width, "GHz"),
            self.photoelectron_pulse.amplitude[None, :],
            u.Quantity(self.photoelectron_pulse.sample_width, "ns"))

        camera = CameraDescription("sstcam", geom, readout)
        optics = OpticsDescription.from_name('SST-ASTRI')
        telescope = TelescopeDescription("SST", "SST", optics, camera)
        subarray = SubarrayDescription(
            'toy',
            tel_positions={1: [0, 0, 0] * u.m},
            tel_descriptions={1: telescope},
        )
        return subarray
Exemplo n.º 2
0
def main():
    std_config = get_standard_config()

    if args.config_file is not None:
        config = replace_config(std_config, read_configuration_file(args.config_file))
    else:
        config = std_config

    print(config['tailcut'])

    geom = CameraGeometry.from_name('LSTCam-002')
    foclen = OpticsDescription.from_name('LST').equivalent_focal_length
    dl1_container = DL1ParametersContainer()
    parameters_to_update = list(HillasParametersContainer().keys())
    parameters_to_update.extend(['wl', 'r', 'leakage', 'n_islands', 'intercept', 'time_gradient'])

    nodes_keys = get_dataset_keys(args.input_file)
    if args.noimage:
        nodes_keys.remove(dl1_images_lstcam_key)

    auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)

    with tables.open_file(args.input_file, mode='r') as input:
        image_table = input.root[dl1_images_lstcam_key]
        with tables.open_file(args.output_file, mode='a') as output:

            params = output.root[dl1_params_lstcam_key].read()

            for ii, row in enumerate(image_table):
                if ii%10000 == 0:
                    print(ii)
                image = row['image']
                pulse_time = row['pulse_time']
                signal_pixels = tailcuts_clean(geom, image, **config['tailcut'])
                if image[signal_pixels].shape[0] > 0:
                    num_islands, island_labels = number_of_islands(geom, signal_pixels)
                    hillas = hillas_parameters(geom[signal_pixels], image[signal_pixels])

                    dl1_container.fill_hillas(hillas)
                    dl1_container.set_timing_features(geom[signal_pixels],
                                                      image[signal_pixels],
                                                      pulse_time[signal_pixels],
                                                      hillas)
                    dl1_container.set_leakage(geom, image, signal_pixels)
                    dl1_container.n_islands = num_islands
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width.value
                    dl1_container.length = length.value
                    dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2)

                    for p in parameters_to_update:
                        params[ii][p] = Quantity(dl1_container[p]).value
                else:
                    for p in parameters_to_update:
                        params[ii][p] = 0

            output.root[dl1_params_lstcam_key][:] = params
Exemplo n.º 3
0
    def _generator(self):

        # container for LST data
        self.data = LSTDataContainer()
        self.data.meta['input_url'] = self.input_url
        self.data.meta['max_events'] = self.max_events
        self.data.meta['origin'] = 'LSTCAM'

        # fill LST data from the CameraConfig table
        self.fill_lst_service_container_from_zfile()

        # Instrument information
        for tel_id in self.data.lst.tels_with_data:

            assert (tel_id == 0
                    or tel_id == 1)  # only LST1 (for the moment id = 0)

            # optics info from standard optics.fits.gz file
            optics = OpticsDescription.from_name("LST")

            # camera info from LSTCam-[geometry_version].camgeom.fits.gz file
            geometry_version = 2
            camera = CameraGeometry.from_name("LSTCam", geometry_version)

            tel_descr = TelescopeDescription(name='LST',
                                             tel_type='LST',
                                             optics=optics,
                                             camera=camera)

            self.n_camera_pixels = tel_descr.camera.n_pixels
            tels = {tel_id: tel_descr}

            # LSTs telescope position taken from MC from the moment
            tel_pos = {tel_id: [50., 50., 16] * u.m}

        subarray = SubarrayDescription("LST1 subarray")
        subarray.tels = tels
        subarray.positions = tel_pos

        self.data.inst.subarray = subarray

        # initialize general monitoring container
        self.initialize_mon_container()

        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count

            # fill specific LST event data
            self.fill_lst_event_container_from_zfile(event)

            # fill general monitoring data
            self.fill_mon_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)

            yield self.data
Exemplo n.º 4
0
    def __init__(self, config=None, tool=None, **kwargs):
        """
        Constructor

        Parameters
        ----------
        config: traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool: ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        kwargs: dict
            Additional parameters to be passed.
            NOTE: The file mask of the data to read can be passed with
            the 'input_url' parameter.
        """

        file_list = glob.glob(kwargs['input_url'])
        file_list.sort()

        # EventSource can not handle file wild cards as input_url
        # To overcome this we substitute the input_url with first file matching
        # the specified file mask.
        del kwargs['input_url']
        super().__init__(config=config, tool=tool, input_url=file_list[0], **kwargs)

        try:
            import uproot
        except ImportError:
            msg = "The `uproot` python module is required to access the MAGIC data"
            self.log.error(msg)
            raise

        # Retrieving the list of run numbers corresponding to the data files
        run_numbers = list(map(self._get_run_number, file_list))
        self.run_numbers = np.unique(run_numbers)

        # # Setting up the current run with the first run present in the data
        # self.current_run = self._set_active_run(run_number=0)
        self.current_run = None
        
        # MAGIC telescope positions in m wrt. to the center of CTA simulations
        self.magic_tel_positions = {
            1: [-27.24, -146.66, 50.00] * u.m,
            2: [-96.44, -96.77, 51.00] * u.m
        }
        # MAGIC telescope description
        optics = OpticsDescription.from_name('MAGIC')
        geom = CameraGeometry.from_name('MAGICCam')
        self.magic_tel_description = TelescopeDescription(optics=optics, camera=geom)
        self.magic_tel_descriptions = {1: self.magic_tel_description, 2: self.magic_tel_description}
        self.magic_subarray = SubarrayDescription('MAGIC', self.magic_tel_positions, self.magic_tel_descriptions)
Exemplo n.º 5
0
    def _generator(self):

        # container for NectarCAM data
        self.data = NectarCAMDataContainer()
        self.data.meta['input_url'] = self.input_url

        # fill data from the CameraConfig table
        self.fill_nectarcam_service_container_from_zfile()


        # Instrument information
        for tel_id in self.data.nectarcam.tels_with_data:
            assert (tel_id == 0)  # only one telescope for the moment (id = 0)

            # optics info from standard optics.fits.gz file
            optics = OpticsDescription.from_name("MST")
            optics.tel_subtype = ''  # to correct bug in reading

            # camera info from NectarCam-[geometry_version].camgeom.fits.gz file
            geometry_version = 2
            camera = CameraGeometry.from_name("NectarCam", geometry_version)

            tel_descr = TelescopeDescription(optics, camera)

            tel_descr.optics.tel_subtype = ''  # to correct bug in reading

            self.n_camera_pixels = tel_descr.camera.n_pixels
            tels = {tel_id: tel_descr}

            # LSTs telescope position
            tel_pos = {tel_id: [0., 0., 0] * u.m}


        self.subarray = SubarrayDescription("MST prototype subarray")
        self.subarray.tels = tels
        self.subarray.positions = tel_pos

        self.data.inst.subarray = self.subarray



        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count

            # fill specific NectarCAM event data
            self.fill_nectarcam_event_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)
            yield self.data
Exemplo n.º 6
0
    def _generator(self):

        # container for LST data
        self.data = LSTDataContainer()
        self.data.meta['input_url'] = self.input_url
        self.data.meta['max_events'] = self.max_events


        # fill LST data from the CameraConfig table
        self.fill_lst_service_container_from_zfile()

        # Instrument information
        for tel_id in self.data.lst.tels_with_data:

            assert (tel_id == 0)  # only LST1 for the moment (id = 0)

            # optics info from standard optics.fits.gz file
            optics = OpticsDescription.from_name("LST")
            optics.tel_subtype = ''  # to correct bug in reading

            # camera info from LSTCam-[geometry_version].camgeom.fits.gz file
            geometry_version = 2
            camera = CameraGeometry.from_name("LSTCam", geometry_version)

            tel_descr = TelescopeDescription(optics, camera)

            self.n_camera_pixels = tel_descr.camera.n_pixels
            tels = {tel_id: tel_descr}

            # LSTs telescope position taken from MC from the moment
            tel_pos = {tel_id: [50., 50., 16] * u.m}


        subarray = SubarrayDescription("LST1 subarray")
        subarray.tels = tels
        subarray.positions = tel_pos

        self.data.inst.subarray = subarray

        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count

            # fill specific LST event data
            self.fill_lst_event_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)
            yield self.data
Exemplo n.º 7
0
    def _generator(self):

        # container for NectarCAM data
        self.data = NectarCAMDataContainer()
        self.data.meta['input_url'] = self.input_url

        # fill data from the CameraConfig table
        self.fill_nectarcam_service_container_from_zfile()

        # Instrument information
        for tel_id in self.data.nectarcam.tels_with_data:
            assert (tel_id == 0)  # only one telescope for the moment (id = 0)

            # optics info from standard optics.fits.gz file
            optics = OpticsDescription.from_name("MST")
            optics.tel_subtype = ''  # to correct bug in reading

            # camera info from NectarCam-[geometry_version].camgeom.fits.gz file
            geometry_version = 2
            camera = CameraGeometry.from_name("NectarCam", geometry_version)

            tel_descr = TelescopeDescription(optics, camera)

            tel_descr.optics.tel_subtype = ''  # to correct bug in reading

            self.n_camera_pixels = tel_descr.camera.n_pixels
            tels = {tel_id: tel_descr}

            # LSTs telescope position
            tel_pos = {tel_id: [0., 0., 0] * u.m}

        self.subarray = SubarrayDescription("MST prototype subarray")
        self.subarray.tels = tels
        self.subarray.positions = tel_pos

        self.data.inst.subarray = self.subarray

        # loop on events
        for count, event in enumerate(self.multi_file):

            self.data.count = count

            # fill specific NectarCAM event data
            self.fill_nectarcam_event_container_from_zfile(event)

            # fill general R0 data
            self.fill_r0_container_from_zfile(event)
            yield self.data
Exemplo n.º 8
0
def get_expected_source_pos(data, data_type, config):
    """Get expected source position for source-dependent analysis .

    Parameters:
    -----------
    data: Pandas DataFrame
    data_type: 'mc_gamma','mc_proton','real_data'
    config: dictionnary containing configuration
    
    """

    #For gamma MC, expected source position is actual one for each event
    if data_type == 'mc_gamma':
        expected_src_pos_x_m = data['src_x'].values
        expected_src_pos_y_m = data['src_y'].values

    #For proton MC, nominal source position is one written in config file
    if data_type == 'mc_proton':

        focal_length = OpticsDescription.from_name(
            'LST').equivalent_focal_length

        expected_src_pos = utils.sky_to_camera(
            u.Quantity(data['mc_alt_tel'].values +
                       config['mc_nominal_source_x_deg'],
                       u.deg,
                       copy=False),
            u.Quantity(data['mc_az_tel'].values +
                       config['mc_nominal_source_y_deg'],
                       u.deg,
                       copy=False), focal_length,
            u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False),
            u.Quantity(data['mc_az_tel'].values, u.deg, copy=False))

        expected_src_pos_x_m = expected_src_pos.x.to_value()
        expected_src_pos_y_m = expected_src_pos.y.to_value()

    # For real data
    # TODO: expected source position for real data should be obtained by using tel_alt,az and source RA,Dec
    # For the moment, expected source position is defined as camera center (ON mode)
    if data_type == 'real_data':
        expected_src_pos_x_m = np.zeros(len(data))
        expected_src_pos_y_m = np.zeros(len(data))

    return expected_src_pos_x_m, expected_src_pos_y_m
Exemplo n.º 9
0
    def prepare_subarray_info(self, tel_id=0):
        """
        Constructs a SubarrayDescription object.
        Parameters
        ----------
        tel_id: int
            Telescope identifier.
        Returns
        -------
        SubarrayDescription :
            instrumental information
        """
        tel_descriptions = {}  # tel_id : TelescopeDescription
        tel_positions = {}  # tel_id : TelescopeDescription

        # optics info from standard optics.fits.gz file
        optics = OpticsDescription.from_name("MST")
        optics.tel_subtype = ''  # to correct bug in reading

        # camera info from NectarCam-[geometry_version].camgeom.fits.gz file
        camera = CameraGeometry.from_name("NectarCam", self.geometry_version)

        tel_descr = TelescopeDescription(name='MST',
                                         tel_type='NectarCam',
                                         optics=optics,
                                         camera=camera)
        tel_descr.optics.tel_subtype = ''  # to correct bug in reading

        self.n_camera_pixels = tel_descr.camera.n_pixels

        # MST telescope position
        tel_positions[tel_id] = [0., 0., 0] * u.m
        tel_descriptions[tel_id] = tel_descr

        return SubarrayDescription(
            "Adlershof",
            tel_positions=tel_positions,
            tel_descriptions=tel_descriptions,
        )
Exemplo n.º 10
0
import pandas as pd
from astropy.table import Table
import RFGHsep_fromfile as rf

#Read data into pandas DataFrame
filetype = 'hdf5'
gammafile = "/scratch/bernardos/LST1/Events/gamma_events_diff.hdf5"  #File with events
protonfile = "/scratch/bernardos/LST1/Events/proton_events.hdf5"  #File with events
dat_gamma = Table.read(gammafile, format=filetype, path='gamma')
dat_proton = Table.read(protonfile, format=filetype, path='proton')

df_gamma = dat_gamma.to_pandas()
df_proton = dat_proton.to_pandas()

#Get some telescope parameters
tel = OpticsDescription.from_name('LST')  #Telescope description
focal_length = tel.equivalent_focal_length.value  #Telescope focal length

#Calculate source position and Disp distance:
sourcepos_gamma = Disp.calc_CamSourcePos(df_gamma['mcAlt'].get_values(),
                                         df_gamma['mcAz'].get_values(),
                                         df_gamma['mcAlttel'].get_values(),
                                         df_gamma['mcAztel'].get_values(),
                                         focal_length)
disp_gamma = Disp.calc_DISP(sourcepos_gamma[0], sourcepos_gamma[1],
                            df_gamma['x'].get_values(),
                            df_gamma['y'].get_values())

sourcepos_proton = Disp.calc_CamSourcePos(df_proton['mcAlt'].get_values(),
                                          df_proton['mcAz'].get_values(),
                                          df_proton['mcAlttel'].get_values(),
Exemplo n.º 11
0
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
              output_filename=None,
              custom_config={},
              pedestal_path=None,
              calibration_path=None,
              time_calibration_path=None,
              pointing_file_path=None):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str
        path to output file, default: `./` + basename(input_filename)
    config_file: path to a configuration file
    pointing_file_path: path to the Drive log with the pointing information

    Returns
    -------

    """
    if output_filename is None:
        output_filename = ('dl1_' +
                           os.path.basename(input_filename).split('.')[0] +
                           '.h5')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]

    try:
        source = event_source(input_filename, back_seekable=True)
    except:
        # back_seekable might not be available for other sources that eventio
        # TODO for real data: source with calibration file and pointing file
        source = event_source(input_filename)

    is_simu = source.metadata['is_simulation']

    source.allowed_tels = config["allowed_tels"]
    source.max_events = config["max_events"]

    metadata = global_metadata(source)
    write_metadata(metadata, output_filename)

    cal = load_calibrator_from_config(config)

    if not is_simu:
        # TODO : add calibration config in config file, read it and pass it here

        r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path,
                                            tel_id=1)

        # all this will be cleaned up in a next PR related to the configuration files
        r1_dl1_calibrator = LSTCameraCalibrator(
            calibration_path=calibration_path,
            time_calibration_path=time_calibration_path,
            image_extractor=config['image_extractor'],
            gain_threshold=Config(config).gain_selector_config['threshold'],
            config=Config(config),
            allowed_tels=[1],
        )

    dl1_container = DL1ParametersContainer()

    if pointing_file_path:
        # Open drive report
        pointings = PointingPosition()
        pointings.drive_path = pointing_file_path
        drive_data = pointings._read_drive_report()

    extra_im = ExtraImageInfo()
    extra_im.prefix = ''  # get rid of the prefix

    event = next(iter(source))

    write_array_info(event, output_filename)
    ### Write extra information to the DL1 file
    if is_simu:
        write_mcheader(event.mcheader,
                       output_filename,
                       obs_id=event.r0.obs_id,
                       filters=filters,
                       metadata=metadata)
        subarray = event.inst.subarray

    with HDF5TableWriter(
            filename=output_filename,
            group_name='dl1/event',
            mode='a',
            filters=filters,
            add_prefix=True,
            # overwrite=True,
    ) as writer:

        print("USING FILTERS: ", writer._h5file.filters)

        if is_simu:
            # build a mapping of tel_id back to tel_index:
            # (note this should be part of SubarrayDescription)
            idx = np.zeros(max(subarray.tel_indices) + 1)
            for key, val in subarray.tel_indices.items():
                idx[key] = val

            # the final transform then needs the mapping and the number of telescopes
            tel_list_transform = partial(
                utils.expand_tel_list,
                max_tels=len(event.inst.subarray.tel) + 1,
            )

            writer.add_column_transform(table_name='subarray/trigger',
                                        col_name='tels_with_trigger',
                                        transform=tel_list_transform)

        ### EVENT LOOP ###
        for i, event in enumerate(source):
            if i % 100 == 0:
                print(i)

            event.dl0.prefix = ''
            event.mc.prefix = 'mc'
            event.trig.prefix = ''

            # write sub tables
            if is_simu:
                write_subarray_tables(writer, event, metadata)

            if not custom_calibration and is_simu:
                cal(event)

            if not is_simu:
                r0_r1_calibrator.calibrate(event)
                r1_dl1_calibrator(event)

            for ii, telescope_id in enumerate(event.r0.tels_with_data):

                tel = event.dl1.tel[telescope_id]
                tel.prefix = ''  # don't really need one
                # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
                tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
                tel_name = tel_name.replace('-003', '')

                if custom_calibration:
                    lst_calibration(event, telescope_id)

                try:
                    dl1_filled = get_dl1(event,
                                         telescope_id,
                                         dl1_container=dl1_container,
                                         custom_config=config,
                                         use_main_island=True)

                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')
                    continue

                if dl1_filled is not None:

                    # Some custom def
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    # Log10(Energy) in GeV
                    if is_simu:
                        dl1_container.mc_energy = event.mc.energy.value
                        dl1_container.log_mc_energy = np.log10(
                            event.mc.energy.value * 1e3)
                        dl1_container.fill_mc(event)

                    dl1_container.log_intensity = np.log10(
                        dl1_container.intensity)
                    dl1_container.gps_time = event.trig.gps_time.value

                    if not is_simu:
                        # GPS time is not available for the time being. Meanwhile,
                        # timestamps can be extracted from the UCTS and alternatively
                        # calculated from the TIB/Dragon modules counters + NTP time
                        # corresponding to the start of the run. For the time being,
                        # we will store the three of them in the dl1 files.
                        # This will be deprecated and modified back to directly use
                        # gps_time whenever the GPS starts working reliably.

                        # gps_time = event.r0.tel[telescope_id].trigger_time

                        ucts_time = event.lst.tel[
                            telescope_id].evt.ucts_timestamp * 1e-9  # nsecs

                        # Get counters from the central Dragon module
                        module_id = 82

                        dragon_time = (event.lst.tel[telescope_id].svc.date +
                                       event.lst.tel[telescope_id].evt.
                                       pps_counter[module_id] +
                                       event.lst.tel[telescope_id].evt.
                                       tenMHz_counter[module_id] * 10**(-7))

                        tib_time = (
                            event.lst.tel[telescope_id].svc.date +
                            event.lst.tel[telescope_id].evt.tib_pps_counter +
                            event.lst.tel[telescope_id].evt.tib_tenMHz_counter
                            * 10**(-7))

                        #dl1_container.gps_time = gps_time
                        dl1_container.tib_time = tib_time
                        dl1_container.ucts_time = ucts_time
                        dl1_container.dragon_time = dragon_time

                        # Select the timestamps to be used for pointing interpolation
                        if config['timestamps_pointing'] == "ucts":
                            event_timestamps = ucts_time
                        elif config['timestamps_pointing'] == "dragon":
                            event_timestamps = dragon_time
                        elif config['timestamps_pointing'] == "tib":
                            event_timestamps = tib_time
                        else:
                            raise ValueError(
                                "The timestamps_pointing option is not a valid one. \
                                    Try ucts (default), dragon or tib.")

                        if pointing_file_path and event_timestamps > 0:
                            azimuth, altitude = pointings.cal_pointingposition(
                                event_timestamps, drive_data)
                            event.pointing[telescope_id].azimuth = azimuth
                            event.pointing[telescope_id].altitude = altitude
                            dl1_container.az_tel = azimuth
                            dl1_container.alt_tel = altitude

                    foclen = event.inst.subarray.tel[
                        telescope_id].optics.equivalent_focal_length
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(
                        np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width.value
                    dl1_container.length = length.value

                    dl1_container.prefix = tel.prefix

                    extra_im.tel_id = telescope_id
                    extra_im.num_trig_pix = event.r0.tel[
                        telescope_id].num_trig_pix
                    extra_im.trigger_time = event.r0.tel[
                        telescope_id].trigger_time
                    extra_im.trigger_type = event.r0.tel[
                        telescope_id].trigger_type
                    extra_im.trig_pix_id = event.r0.tel[
                        telescope_id].trig_pix_id

                    for container in [extra_im, dl1_container, event.r0, tel]:
                        add_global_metadata(container, metadata)

                    event.r0.prefix = ''

                    writer.write(table_name=f'telescope/image/{tel_name}',
                                 containers=[event.r0, tel, extra_im])
                    writer.write(table_name=f'telescope/parameters/{tel_name}',
                                 containers=[dl1_container, extra_im])

                    # writes mc information per telescope, including photo electron image
                    if is_simu \
                            and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
                            and config['write_pe_image']:
                        event.mc.tel[telescope_id].prefix = ''
                        writer.write(
                            table_name=f'simulation/{tel_name}',
                            containers=[event.mc.tel[telescope_id], extra_im])

    if is_simu:
        ### Reconstruct source position from disp for all events and write the result in the output file
        for tel_name in ['LST_LSTCam']:
            focal = OpticsDescription.from_name(
                tel_name.split('_')[0]).equivalent_focal_length
            dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
            add_disp_to_parameters_table(output_filename, dl1_params_key,
                                         focal)

    # Write energy histogram from simtel file and extra metadata
    if is_simu:
        write_simtel_energy_histogram(source,
                                      output_filename,
                                      obs_id=event.dl0.obs_id,
                                      metadata=metadata)
Exemplo n.º 12
0
def apply_models(dl1, classifier, reg_energy, reg_disp_vector, custom_config={}):
    """Apply previously trained Random Forests to a set of data
    depending on a set of features.

    Parameters:
    -----------
    data: Pandas DataFrame

    features: list

    classifier: Random Forest Classifier
    RF for Gamma/Hadron separation

    reg_energy: Random Forest Regressor
    RF for Energy reconstruction

    reg_disp: Random Forest Regressor
    RF for disp_norm reconstruction

    """

    config = replace_config(standard_config, custom_config)
    regression_features = config["regression_features"]
    classification_features = config["classification_features"]

    dl2 = dl1.copy()
    #Reconstruction of Energy and disp_norm distance
    dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features])
    dl2['reco_energy'] = 10**(dl2['log_reco_energy']-3)
    disp_vector = reg_disp_vector.predict(dl2[regression_features])
    dl2['reco_disp_dx'] = disp_vector[:, 0]
    dl2['reco_disp_dy'] = disp_vector[:, 1]

    #Construction of Source position in camera coordinates from disp_norm distance.

    dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos(dl2.reco_disp_dx,
                                                            dl2.reco_disp_dy,
                                                            dl2.x,
                                                            dl2.y,
                                                            )

    focal_length = OpticsDescription.from_name('LST').equivalent_focal_length
    if 'mc_alt_tel' in dl2.columns:
        alt_tel = dl2['mc_alt_tel'].values
        az_tel = dl2['mc_az_tel'].values
    elif 'alt_tel' in dl2.columns:
        alt_tel = dl2['alt_tel'].values
        az_tel = dl2['az_tel'].values
    else:
        alt_tel = - np.pi/2. * np.ones(len(dl2))
        az_tel = - np.pi/2. * np.ones(len(dl2))


    src_pos_reco = utils.reco_source_position_sky(dl2.x.values * u.m,
                                                  dl2.y.values * u.m,
                                                  dl2.reco_disp_dx.values * u.m,
                                                  dl2.reco_disp_dy.values * u.m,
                                                  focal_length,
                                                  alt_tel * u.rad,
                                                  az_tel * u.rad)

    dl2['reco_alt'] = src_pos_reco.alt.rad
    dl2['reco_az'] = src_pos_reco.az.rad

    dl2['reco_type'] = classifier.predict(dl2[classification_features]).astype(int)
    probs = classifier.predict_proba(dl2[classification_features])[0:, 0]
    dl2['gammaness'] = probs
    return dl2
Exemplo n.º 13
0
def r0_to_dl1(
    input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
    output_filename=None,
    custom_config={},
    pedestal_path=None,
    calibration_path=None,
    pointing_file_path=None,
):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str
        path to output file, default: `./` + basename(input_filename)
    config_file: path to a configuration file
    pointing_file_path: path to the Drive log with the pointing information

    Returns
    -------

    """
    if output_filename is None:
        output_filename = ('dl1_' +
                           os.path.basename(input_filename).split('.')[0] +
                           '.h5')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]

    try:
        source = event_source(input_filename, back_seekable=True)
    except:
        # back_seekable might not be available for other sources that eventio
        # TODO for real data: source with calibration file and pointing file
        source = event_source(input_filename)

    is_simu = source.metadata['is_simulation']

    source.allowed_tels = config["allowed_tels"]
    source.max_events = config["max_events"]

    metadata = global_metadata(source)
    write_metadata(metadata, output_filename)

    cal = load_calibrator_from_config(config)

    if not is_simu:
        # TODO : add calibration config in config file, read it and pass it here
        charge_config = Config(
            {"LocalPeakWindowSum": {
                "window_shift": 5,
                "window_width": 12
            }})

        r0_r1_calibrator = LSTR0Corrections(
            pedestal_path=pedestal_path,
            r1_sample_start=2,  # numbers in config ?
            r1_sample_end=38,
        )
        r1_dl1_calibrator = LSTCameraCalibrator(
            calibration_path=calibration_path,
            image_extractor=config['image_extractor'],
            config=charge_config,
            allowed_tels=[1],
        )

    dl1_container = DL1ParametersContainer()

    if pointing_file_path:
        # Open drive report
        pointings = PointingPosition()
        pointings.drive_path = pointing_file_path
        drive_data = pointings._read_drive_report()

    extra_im = ExtraImageInfo()
    extra_im.prefix = ''  # get rid of the prefix

    event = next(iter(source))
    write_array_info(event, output_filename)
    ### Write extra information to the DL1 file
    if is_simu:
        write_mcheader(event.mcheader,
                       output_filename,
                       obs_id=event.r0.obs_id,
                       filters=filters,
                       metadata=metadata)
        subarray = event.inst.subarray

    with HDF5TableWriter(
            filename=output_filename,
            group_name='dl1/event',
            mode='a',
            filters=filters,
            add_prefix=True,
            # overwrite=True,
    ) as writer:

        print("USING FILTERS: ", writer._h5file.filters)

        if is_simu:
            # build a mapping of tel_id back to tel_index:
            # (note this should be part of SubarrayDescription)
            idx = np.zeros(max(subarray.tel_indices) + 1)
            for key, val in subarray.tel_indices.items():
                idx[key] = val

            # the final transform then needs the mapping and the number of telescopes
            tel_list_transform = partial(
                utils.expand_tel_list,
                max_tels=len(event.inst.subarray.tel) + 1,
            )

            writer.add_column_transform(table_name='subarray/trigger',
                                        col_name='tels_with_trigger',
                                        transform=tel_list_transform)

        ### EVENT LOOP ###
        for i, event in enumerate(source):
            if i % 100 == 0:
                print(i)

            event.dl0.prefix = ''
            event.mc.prefix = 'mc'
            event.trig.prefix = ''

            # write sub tables
            if is_simu:
                write_subarray_tables(writer, event, metadata)

            if not custom_calibration and is_simu:
                cal(event)

            if not is_simu:
                r0_r1_calibrator.calibrate(event)
                r1_dl1_calibrator(event)

            for ii, telescope_id in enumerate(event.r0.tels_with_data):

                if not is_simu:
                    combine_channels(event, telescope_id, 4095)

                tel = event.dl1.tel[telescope_id]
                tel.prefix = ''  # don't really need one
                # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
                tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
                tel_name = tel_name.replace('-002', '')

                if custom_calibration:
                    lst_calibration(event, telescope_id)

                try:
                    dl1_filled = get_dl1(event,
                                         telescope_id,
                                         dl1_container=dl1_container,
                                         custom_config=config,
                                         use_main_island=True)

                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')
                    continue

                if dl1_filled is not None:

                    # Some custom def
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    # Log10(Energy) in GeV
                    if is_simu:
                        dl1_container.mc_energy = event.mc.energy.value
                        dl1_container.log_mc_energy = np.log10(
                            event.mc.energy.value * 1e3)
                        dl1_container.fill_mc(event)

                    dl1_container.log_intensity = np.log10(
                        dl1_container.intensity)
                    dl1_container.gps_time = event.trig.gps_time.value

                    if not is_simu:
                        # For real data, GPS time is not available for the time being.
                        # In the mean time, it is taken from TIB pps and 10 MHz counters
                        # since UCTS timestamps do not seem to be trustable. This will be
                        # deprecated and modified back to directly use gps_time whenever
                        # the GPS starts working.

                        # TAI time in s taken from TIB
                        tai_time = event.r0.tel[telescope_id].trigger_time
                        utc_time = Time(datetime.utcfromtimestamp(tai_time))

                        gps_time = utc_time.gps
                        dl1_container.gps_time = gps_time

                        if pointing_file_path:
                            azimuth, altitude = pointings.cal_pointingposition(
                                utc_time.unix, drive_data)
                            event.pointing[telescope_id].azimuth = azimuth
                            event.pointing[telescope_id].altitude = altitude
                            dl1_container.az_tel = azimuth
                            dl1_container.alt_tel = altitude

                    foclen = event.inst.subarray.tel[
                        telescope_id].optics.equivalent_focal_length
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(
                        np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width.value
                    dl1_container.length = length.value

                    dl1_container.prefix = tel.prefix

                    extra_im.tel_id = telescope_id
                    for container in [extra_im, dl1_container, event.r0, tel]:
                        add_global_metadata(container, metadata)

                    event.r0.prefix = ''

                    writer.write(table_name=f'telescope/image/{tel_name}',
                                 containers=[event.r0, tel, extra_im])
                    writer.write(table_name=f'telescope/parameters/{tel_name}',
                                 containers=[dl1_container])

                    # writes mc information per telescope, including photo electron image
                    if is_simu \
                            and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
                            and config['write_pe_image']:
                        event.mc.tel[telescope_id].prefix = ''
                        writer.write(
                            table_name=f'simulation/{tel_name}',
                            containers=[event.mc.tel[telescope_id], extra_im])

    if is_simu:
        ### Reconstruct source position from disp for all events and write the result in the output file
        for tel_name in ['LST_LSTCam']:
            focal = OpticsDescription.from_name(
                tel_name.split('_')[0]).equivalent_focal_length
            dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
            add_disp_to_parameters_table(output_filename, dl1_params_key,
                                         focal)

    # Write energy histogram from simtel file and extra metadata
    if is_simu:
        write_simtel_energy_histogram(source,
                                      output_filename,
                                      obs_id=event.dl0.obs_id,
                                      metadata=metadata)
    def __init__(self, **kwargs):
        """
        Constructor

        Parameters
        ----------
        kwargs: dict
            Parameters to be passed.
            NOTE: The file mask of the data to read can be passed with
            the 'input_url' parameter.
        """
        try:
            import uproot
        except ImportError:
            raise ImportError(
                "The 'uproot' package is required for the DLMAGICEventSource class."
            )

        self.file_list = glob.glob(kwargs["input_url"])
        self.file_list.sort()

        # Since EventSource can not handle file wild cards as input_url
        # We substitute the input_url with first file matching
        # the specified file mask.
        del kwargs["input_url"]
        super().__init__(input_url=self.file_list[0], **kwargs)

        # Translate MAGIC shower primary id to CTA convention
        self.magic_to_cta_shower_primary_id = {
            1: 0,  # gamma
            14: 101,  # MAGIC proton
            3: 1,  # MAGIC electron
        }
        # MAGIC telescope positions in m wrt. to the center of CTA simulations
        self.magic_tel_positions = {
            1: [-27.24, -146.66, 50.00] * u.m,
            2: [-96.44, -96.77, 51.00] * u.m,
        }
        self.magic_tel_positions = self.magic_tel_positions
        # MAGIC telescope description
        optics = OpticsDescription.from_name("MAGIC")
        geom = CameraGeometry.from_name("MAGICCam")
        # Camera Readout for NectarCam used as a placeholder
        readout = CameraReadout(
            "MAGICCam",
            sampling_rate=u.Quantity(1, u.GHz),
            reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]),
            reference_pulse_sample_width=u.Quantity(1, u.ns),
        )
        camera = CameraDescription("MAGICCam", geom, readout)
        self.magic_tel_description = TelescopeDescription(name="MAGIC",
                                                          tel_type="LST",
                                                          optics=optics,
                                                          camera=camera)
        self.magic_tel_descriptions = {
            1: self.magic_tel_description,
            2: self.magic_tel_description,
        }
        self.magic_subarray = SubarrayDescription("MAGIC",
                                                  self.magic_tel_positions,
                                                  self.magic_tel_descriptions)
        # Open ROOT files
        self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = (
            None,
            None,
            None,
            None,
            None,
        )
        for file in self.file_list:
            uproot_file = uproot.open(file)
            if "_Y_" in file:
                if "_M1_" in file:
                    self.calib_M1 = uproot_file["Events"]
                    self.meta = uproot_file["RunHeaders"]
                elif "_M2_" in file:
                    self.calib_M2 = uproot_file["Events"]
            if "_I_" in file:
                if "_M1_" in file:
                    self.star_M1 = uproot_file["Events"]
                elif "_M2_" in file:
                    self.star_M2 = uproot_file["Events"]
            if "_S_" in file:
                self.superstar = uproot_file["Events"]
                self.meta = uproot_file["RunHeaders"]

        # figure out if MC or Data run
        self.mc = "MMcCorsikaRunHeader." in self.meta.keys()

        # get the run number directly from the root file
        if self.mc:
            self.run_number = int(
                uproot_file["RunHeaders"]["MMcCorsikaRunHeader."]
                ["MMcCorsikaRunHeader.fRunNumber"].array()[0])
        else:
            self.run_number = int(uproot_file["RunHeaders"]["MRawRunHeader_1."]
                                  ["MRawRunHeader_1.fRunNumber"].array()[0])

        self._header = self._parse_header()
Exemplo n.º 15
0
def apply_models(dl1,
                 classifier,
                 reg_energy,
                 reg_disp_vector,
                 custom_config={}):
    """Apply previously trained Random Forests to a set of data
    depending on a set of features.

    Parameters:
    -----------
    data: Pandas DataFrame

    features: list

    classifier: Random Forest Classifier
    RF for Gamma/Hadron separation

    reg_energy: Random Forest Regressor
    RF for Energy reconstruction

    reg_disp: Random Forest Regressor
    RF for disp_norm reconstruction

    """

    config = replace_config(standard_config, custom_config)

    dl2 = dl1.copy()

    regression_features = config["regression_features"]
    classification_features = config["classification_features"]

    #Reconstruction of Energy and disp_norm distance
    dl2['log_reco_energy'] = reg_energy.predict(dl2[regression_features])
    dl2['reco_energy'] = 10**(dl2['log_reco_energy'])
    disp_vector = reg_disp_vector.predict(dl2[regression_features])
    dl2['reco_disp_dx'] = disp_vector[:, 0]
    dl2['reco_disp_dy'] = disp_vector[:, 1]

    #Construction of Source position in camera coordinates from disp_norm distance.

    dl2['reco_src_x'], dl2['reco_src_y'] = disp.disp_to_pos(
        dl2.reco_disp_dx,
        dl2.reco_disp_dy,
        dl2.x,
        dl2.y,
    )

    focal_length = OpticsDescription.from_name('LST').equivalent_focal_length
    if 'mc_alt_tel' in dl2.columns:
        alt_tel = dl2['mc_alt_tel'].values
        az_tel = dl2['mc_az_tel'].values
    elif 'alt_tel' in dl2.columns:
        alt_tel = dl2['alt_tel'].values
        az_tel = dl2['az_tel'].values
    else:
        alt_tel = -np.pi / 2. * np.ones(len(dl2))
        az_tel = -np.pi / 2. * np.ones(len(dl2))

    src_pos_reco = utils.reco_source_position_sky(
        dl2.x.values * u.m, dl2.y.values * u.m, dl2.reco_disp_dx.values * u.m,
        dl2.reco_disp_dy.values * u.m, focal_length, alt_tel * u.rad,
        az_tel * u.rad)

    dl2['reco_alt'] = src_pos_reco.alt.rad
    dl2['reco_az'] = src_pos_reco.az.rad

    dl2['reco_type'] = classifier.predict(
        dl2[classification_features]).astype(int)
    probs = classifier.predict_proba(dl2[classification_features])

    # This check is valid as long as we train on only two classes (gammas and protons)
    if probs.shape[1] > 2:
        raise ValueError(
            "The classifier is predicting more than two classes, "
            "the predicted probabilty to assign as gammaness is unclear."
            "Please check training data")

    # gammaness is the prediction probability for the first class (0)
    dl2['gammaness'] = probs[:, 0]

    return dl2
Exemplo n.º 16
0
def main():
    std_config = get_standard_config()

    log.setLevel(logging.INFO)
    handler = logging.StreamHandler()
    logging.getLogger().addHandler(handler)

    if args.config_file is not None:
        config = replace_config(std_config, read_configuration_file(args.config_file))
    else:
        config = std_config

    log.info(f"Tailcut config used: {config['tailcut']}")

    foclen = OpticsDescription.from_name('LST').equivalent_focal_length
    cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam")
    camera_geom = CameraGeometry.from_table(cam_table)

    dl1_container = DL1ParametersContainer()
    parameters_to_update = list(HillasParametersContainer().keys())
    parameters_to_update.extend([
        'concentration_cog',
        'concentration_core',
        'concentration_pixel',
        'leakage_intensity_width_1',
        'leakage_intensity_width_2',
        'leakage_pixels_width_1',
        'leakage_pixels_width_2',
        'n_islands',
        'intercept',
        'time_gradient',
        'n_pixels',
        'wl',
        'log_intensity'
    ])

    nodes_keys = get_dataset_keys(args.input_file)
    if args.noimage:
        nodes_keys.remove(dl1_images_lstcam_key)

    auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)

    with tables.open_file(args.input_file, mode='r') as input:
        image_table = input.root[dl1_images_lstcam_key]
        dl1_params_input = input.root[dl1_params_lstcam_key].colnames
        disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'}
        if set(dl1_params_input).intersection(disp_params):
            parameters_to_update.extend(disp_params)

        with tables.open_file(args.output_file, mode='a') as output:
            params = output.root[dl1_params_lstcam_key].read()
            for ii, row in enumerate(image_table):

                dl1_container.reset()

                image = row['image']
                peak_time = row['peak_time']

                signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut'])

                n_pixels = np.count_nonzero(signal_pixels)
                if n_pixels > 0:
                    num_islands, island_labels = number_of_islands(camera_geom, signal_pixels)
                    n_pixels_on_island = np.bincount(island_labels.astype(np.int))
                    n_pixels_on_island[0] = 0  # first island is no-island and should not be considered
                    max_island_label = np.argmax(n_pixels_on_island)
                    signal_pixels[island_labels != max_island_label] = False

                    hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels])

                    dl1_container.fill_hillas(hillas)
                    dl1_container.set_timing_features(camera_geom[signal_pixels],
                                                      image[signal_pixels],
                                                      peak_time[signal_pixels],
                                                      hillas)

                    dl1_container.set_leakage(camera_geom, image, signal_pixels)
                    dl1_container.set_concentration(camera_geom, image, hillas)
                    dl1_container.n_islands = num_islands
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    dl1_container.n_pixels = n_pixels
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width
                    dl1_container.length = length
                    dl1_container.log_intensity = np.log10(dl1_container.intensity)

                if set(dl1_params_input).intersection(disp_params):
                    disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
                        dl1_container['x'].to_value(u.m),
                        dl1_container['y'].to_value(u.m),
                        params['src_x'][ii],
                        params['src_y'][ii]
                    )

                    dl1_container['disp_dx'] = disp_dx
                    dl1_container['disp_dy'] = disp_dy
                    dl1_container['disp_norm'] = disp_norm
                    dl1_container['disp_angle'] = disp_angle
                    dl1_container['disp_sign'] = disp_sign

                for p in parameters_to_update:
                    params[ii][p] = u.Quantity(dl1_container[p]).value

            output.root[dl1_params_lstcam_key][:] = params
Exemplo n.º 17
0
def r0_to_dl1(
    input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
    output_filename=None,
    custom_config={},
):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str or None
        path to output file, defaults to writing dl1 into the current directory
    custom_config: path to a configuration file

    Returns
    -------

    """

    if output_filename is None:
        try:
            run = parse_r0_filename(input_filename)
            output_filename = run_to_dl1_filename(run.tel_id, run.run,
                                                  run.subrun)
        except ValueError:
            output_filename = r0_to_dl1_filename(Path(input_filename).name)

    if os.path.exists(output_filename):
        raise IOError(str(output_filename) + ' exists, exiting.')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]

    source = EventSource(input_url=input_filename,
                         config=Config(config["source_config"]))
    subarray = source.subarray
    is_simu = source.is_simulation

    metadata = global_metadata(source)
    write_metadata(metadata, output_filename)

    cal_mc = load_calibrator_from_config(config, subarray)

    # minimum number of pe in a pixel to include it
    # in calculation of muon ring time (peak sample):
    min_pe_for_muon_t_calc = 10.

    # Dictionary to store muon ring parameters
    muon_parameters = create_muon_table()

    # all this will be cleaned up in a next PR related to the configuration files
    r1_dl1_calibrator = CameraCalibrator(
        image_extractor_type=config['image_extractor'],
        config=Config(config),
        subarray=subarray)

    if not is_simu:

        # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
        # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
        r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
            image_extractor_type=config['image_extractor_for_muons'],
            config=Config(config),
            subarray=subarray)

        # Component to process interleaved pedestal and flat-fields
        calib_config = Config(config[config['calibration_product']])

        calibration_calculator = CalibrationCalculator.from_name(
            config['calibration_product'],
            config=calib_config,
            subarray=source.subarray)

    calibration_index = DL1MonitoringEventIndexContainer()

    dl1_container = DL1ParametersContainer()

    extra_im = ExtraImageInfo()
    extra_im.prefix = ''  # get rid of the prefix

    # Write extra information to the DL1 file
    write_array_info(subarray, output_filename)
    write_array_info_08(subarray, output_filename)

    if is_simu:
        write_mcheader(
            source.simulation_config,
            output_filename,
            obs_id=source.obs_ids[0],
            filters=HDF5_ZSTD_FILTERS,
            metadata=metadata,
        )

    with HDF5TableWriter(
            filename=output_filename,
            group_name='dl1/event',
            mode='a',
            filters=HDF5_ZSTD_FILTERS,
            add_prefix=True,
            # overwrite=True,
    ) as writer:

        if is_simu:
            subarray = subarray
            # build a mapping of tel_id back to tel_index:
            # (note this should be part of SubarrayDescription)
            idx = np.zeros(max(subarray.tel_indices) + 1)
            for key, val in subarray.tel_indices.items():
                idx[key] = val

            # the final transform then needs the mapping and the number of telescopes
            tel_list_transform = partial(
                utils.expand_tel_list,
                max_tels=max(subarray.tel) + 1,
            )

            writer.add_column_transform(table_name='subarray/trigger',
                                        col_name='tels_with_trigger',
                                        transform=tel_list_transform)

        # Forcing filters for the dl1 dataset that are currently read from the pre-existing files
        # This should be fixed in ctapipe and then corrected here
        writer._h5file.filters = HDF5_ZSTD_FILTERS
        logger.info(f"USING FILTERS: {writer._h5file.filters}")

        for i, event in enumerate(source):

            if i % 100 == 0:
                logger.info(i)

            event.dl0.prefix = ''
            event.trigger.prefix = ''
            if event.simulation is not None:
                event.simulation.prefix = 'mc'

            dl1_container.reset()

            # write sub tables
            if is_simu:
                write_subarray_tables(writer, event, metadata)
                if not custom_calibration:
                    cal_mc(event)
                if config['mc_image_scaling_factor'] != 1:
                    rescale_dl1_charge(event,
                                       config['mc_image_scaling_factor'])

            else:
                if i == 0:
                    # initialize the telescope
                    # FIXME? LST calibrator is only for one telescope
                    # it should be inside the telescope loop (?)

                    tel_id = calibration_calculator.tel_id

                    #initialize the event monitoring data
                    event.mon = source.r0_r1_calibrator.mon_data

                    # write the first calibration event (initialized from calibration h5 file)
                    write_calibration_data(writer,
                                           calibration_index,
                                           event.mon.tel[tel_id],
                                           new_ped=True,
                                           new_ff=True)

                # flat-field or pedestal:
                if (event.trigger.event_type == EventType.FLATFIELD
                        or event.trigger.event_type == EventType.SKY_PEDESTAL):

                    # process interleaved events (pedestals, ff, calibration)
                    new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
                        event)

                    # write monitoring containers if updated
                    if new_ped_event or new_ff_event:
                        write_calibration_data(writer,
                                               calibration_index,
                                               event.mon.tel[tel_id],
                                               new_ped=new_ped_event,
                                               new_ff=new_ff_event)

                    # calibrate and gain select the event by hand for DL1
                    source.r0_r1_calibrator.calibrate(event)

            # create image for all events
            r1_dl1_calibrator(event)

            # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
            # When the last version of the method is implemented, vol. reduction will be done at dl0
            apply_volume_reduction(event, subarray, config)

            # FIXME? This should be eventually done after we evaluate whether the image is
            # a candidate muon ring. In that case the full image could be kept, or reduced
            # only after the ring analysis is complete.

            for ii, telescope_id in enumerate(event.dl1.tel.keys()):

                dl1_container.reset()

                # update the calibration index in the dl1 event container
                dl1_container.calibration_id = calibration_index.calibration_id

                dl1_container.fill_event_info(event)

                tel = event.dl1.tel[telescope_id]
                tel.prefix = ''  # don't really need one
                # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
                tel_name = str(subarray.tel[telescope_id])[4:]

                if custom_calibration:
                    lst_calibration(event, telescope_id)

                write_event = True
                # Will determine whether this event has to be written to the
                # DL1 output or not.

                if is_simu:
                    dl1_container.fill_mc(event,
                                          subarray.positions[telescope_id])

                assert event.dl1.tel[telescope_id].image is not None

                try:
                    get_dl1(
                        event,
                        subarray,
                        telescope_id,
                        dl1_container=dl1_container,
                        custom_config=config,
                    )

                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')

                if not is_simu:
                    dl1_container.ucts_time = 0
                    # convert Time to unix timestamp in (UTC) to keep compatibility
                    # with older lstchain
                    # FIXME: just keep it as time, table writer and reader handle it
                    dl1_container.dragon_time = event.trigger.time.unix
                    dl1_container.tib_time = 0

                    dl1_container.ucts_trigger_type = event.lst.tel[
                        telescope_id].evt.ucts_trigger_type
                    dl1_container.trigger_type = event.lst.tel[
                        telescope_id].evt.tib_masked_trigger
                else:
                    dl1_container.trigger_type = event.trigger.event_type

                dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
                dl1_container.alt_tel = event.pointing.tel[
                    telescope_id].altitude

                dl1_container.trigger_time = event.trigger.time.unix
                dl1_container.event_type = event.trigger.event_type

                # FIXME: no need to read telescope characteristics like foclen for every event!
                foclen = subarray.tel[
                    telescope_id].optics.equivalent_focal_length
                mirror_area = u.Quantity(
                    subarray.tel[telescope_id].optics.mirror_area, u.m**2)
                dl1_container.prefix = tel.prefix

                # extra info for the image table
                extra_im.tel_id = telescope_id
                extra_im.selected_gain_channel = event.r1.tel[
                    telescope_id].selected_gain_channel

                for container in [extra_im, dl1_container, event.r0, tel]:
                    add_global_metadata(container, metadata)

                event.r0.prefix = ''

                writer.write(table_name=f'telescope/image/{tel_name}',
                             containers=[event.index, tel, extra_im])
                writer.write(table_name=f'telescope/parameters/{tel_name}',
                             containers=[event.index, dl1_container])

                # Muon ring analysis, for real data only (MC is done starting from DL1 files)
                if not is_simu:
                    bad_pixels = event.mon.tel[
                        telescope_id].calibration.unusable_pixels[0]
                    # Set to 0 unreliable pixels:
                    image = tel.image * (~bad_pixels)

                    # process only promising events, in terms of # of pixels with large signals:
                    if tag_pix_thr(image):

                        # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
                        numsamples = event.r1.tel[telescope_id].waveform.shape[
                            1]  # not necessarily the same as in r0!
                        bad_pixels_hg = event.mon.tel[
                            telescope_id].calibration.unusable_pixels[0]
                        bad_pixels_lg = event.mon.tel[
                            telescope_id].calibration.unusable_pixels[1]
                        # Now set to 0 all samples in unreliable pixels. Important for global peak
                        # integrator in case of crazy pixels!  TBD: can this be done in a simpler
                        # way?
                        bad_pixels = bad_pixels_hg | bad_pixels_lg
                        bad_waveform = np.transpose(
                            np.array(numsamples * [bad_pixels]))

                        # print('hg bad pixels:',np.where(bad_pixels_hg))
                        # print('lg bad pixels:',np.where(bad_pixels_lg))

                        event.r1.tel[telescope_id].waveform *= ~bad_waveform
                        r1_dl1_calibrator_for_muon_rings(event)

                        tel = event.dl1.tel[telescope_id]
                        image = tel.image * (~bad_pixels)

                        # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
                        # perhaps the event is no longer promising (e.g. if it has a large time evolution)
                        if not tag_pix_thr(image):
                            good_ring = False
                        else:
                            # read geometry from event.inst. But not needed for every event. FIXME?
                            geom = subarray.tel[telescope_id].\
                                camera.geometry

                            muonintensityparam, dist_mask, \
                            ring_size, size_outside_ring, muonringparam, \
                            good_ring, radial_distribution, \
                            mean_pixel_charge_around_ring,\
                            muonpars = \
                                analyze_muon_event(subarray,
                                                   event.index.event_id,
                                                   image, geom, foclen,
                                                   mirror_area, False, '')
                            #                      mirror_area, True, './')
                            #           (test) plot muon rings as png files

                            # Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
                            bright_pixels = image > min_pe_for_muon_t_calc
                            selected_gain = event.r1.tel[
                                telescope_id].selected_gain_channel
                            mask_hg = bright_pixels & (selected_gain == 0)
                            mask_lg = bright_pixels & (selected_gain == 1)

                            bright_pixels_waveforms_hg = event.r1.tel[
                                telescope_id].waveform[mask_hg, :]
                            bright_pixels_waveforms_lg = event.r1.tel[
                                telescope_id].waveform[mask_lg, :]
                            stacked_waveforms_hg = np.sum(
                                bright_pixels_waveforms_hg, axis=0)
                            stacked_waveforms_lg = np.sum(
                                bright_pixels_waveforms_lg, axis=0)

                            # stacked waveforms from all bright pixels; shape (ngains, nsamples)
                            hg_peak_sample = np.argmax(stacked_waveforms_hg,
                                                       axis=-1)
                            lg_peak_sample = np.argmax(stacked_waveforms_lg,
                                                       axis=-1)

                        if good_ring:
                            fill_muon_event(-1, muon_parameters, good_ring,
                                            event.index.event_id,
                                            dl1_container.dragon_time,
                                            muonintensityparam, dist_mask,
                                            muonringparam, radial_distribution,
                                            ring_size, size_outside_ring,
                                            mean_pixel_charge_around_ring,
                                            muonpars, hg_peak_sample,
                                            lg_peak_sample)

                # writes mc information per telescope, including photo electron image
                if (is_simu and config['write_pe_image']
                        and event.simulation.tel[telescope_id].true_image
                        is not None and
                        event.simulation.tel[telescope_id].true_image.any()):
                    event.simulation.tel[telescope_id].prefix = ''
                    writer.write(table_name=f'simulation/{tel_name}',
                                 containers=[
                                     event.simulation.tel[telescope_id],
                                     extra_im
                                 ])

        if not is_simu:
            # at the end of event loop ask calculation of remaining interleaved statistics
            new_ped, new_ff = calibration_calculator.output_interleaved_results(
                event)
            # write monitoring events
            write_calibration_data(writer,
                                   calibration_index,
                                   event.mon.tel[tel_id],
                                   new_ped=new_ped,
                                   new_ff=new_ff)

    if is_simu:
        # Reconstruct source position from disp for all events and write the result in the output file
        for tel_name in ['LST_LSTCam']:
            focal = OpticsDescription.from_name(
                tel_name.split('_')[0]).equivalent_focal_length
            add_disp_to_parameters_table(output_filename,
                                         dl1_params_lstcam_key, focal)

        # Write energy histogram from simtel file and extra metadata
        # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
        if source.max_events is None:
            write_simtel_energy_histogram(source,
                                          output_filename,
                                          obs_id=event.index.obs_id,
                                          metadata=metadata)
    else:
        dir, name = os.path.split(output_filename)
        name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
        # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
        name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
        muon_output_filename = Path(dir, name)
        table = Table(muon_parameters)
        table.write(muon_output_filename, format='fits', overwrite=True)
Exemplo n.º 18
0
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
              output_filename=None,
              custom_config={},
              pedestal_path=None,
              calibration_path=None,
              time_calibration_path=None,
              pointing_file_path=None,
              ucts_t0_dragon=math.nan,
              dragon_counter0=math.nan,
              ucts_t0_tib=math.nan,
              tib_counter0=math.nan):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str or None
        path to output file, defaults to writing dl1 into the current directory
    custom_config: path to a configuration file
    pedestal_path: Path to the DRS4 pedestal file
    calibration_path: Path to the file with calibration constants and
        pedestals
    time_calibration_path: Path to the DRS4 time correction file
    pointing_file_path: path to the Drive log with the pointing information
    ucts_t0_dragon: first valid ucts_time
    dragon_counter0: Dragon counter corresponding to ucts_t0_dragon
    ucts_t0_tib: first valid ucts_time for the first valid TIB counter
    tib_counter0: first valid TIB counter

    Returns
    -------

    """
    if output_filename is None:
        try:
            run = parse_r0_filename(input_filename)
            output_filename = run_to_dl1_filename(run.tel_id, run.run,
                                                  run.subrun)
        except ValueError:
            output_filename = r0_to_dl1_filename(Path(input_filename).name)

    if os.path.exists(output_filename):
        raise IOError(str(output_filename) + ' exists, exiting.')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]
    gain_selector = load_gain_selector_from_config(config)

    # FIXME for ctapipe 0.8, str should be removed, as Path is supported
    source = event_source(str(input_filename))
    subarray = source.subarray

    is_simu = source.is_simulation

    source.allowed_tels = config["allowed_tels"]
    if config["max_events"] is not None:
        source.max_events = config["max_events"]

    metadata = global_metadata(source)
    write_metadata(metadata, output_filename)

    cal_mc = load_calibrator_from_config(config, subarray)

    # minimum number of pe in a pixel to include it
    # in calculation of muon ring time (peak sample):
    min_pe_for_muon_t_calc = 10.

    # Dictionary to store muon ring parameters
    muon_parameters = create_muon_table()

    if not is_simu:

        # TODO : add DRS4 calibration config in config file, read it and pass it here
        r0_r1_calibrator = LSTR0Corrections(
            pedestal_path=pedestal_path,
            tel_id=1,
        )

        # all this will be cleaned up in a next PR related to the configuration files

        r1_dl1_calibrator = LSTCameraCalibrator(
            calibration_path=calibration_path,
            time_calibration_path=time_calibration_path,
            extractor_product=config['image_extractor'],
            gain_threshold=Config(config).gain_selector_config['threshold'],
            charge_scale=config['charge_scale'],
            config=Config(config),
            allowed_tels=[1],
            subarray=subarray)

        # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
        # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
        r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator(
            calibration_path=calibration_path,
            time_calibration_path=time_calibration_path,
            extractor_product=config['image_extractor_for_muons'],
            gain_threshold=Config(config).gain_selector_config['threshold'],
            charge_scale=config['charge_scale'],
            config=Config(config),
            allowed_tels=[1],
            subarray=subarray)

        # Component to process interleaved pedestal and flat-fields
        calib_config = Config(config[config['calibration_product']])

        # set time calibration path for flatfield trailet ()
        calib_config.FlasherFlatFieldCalculator.time_calibration_path = time_calibration_path

        calibration_calculator = CalibrationCalculator.from_name(
            config['calibration_product'],
            config=calib_config,
            subarray=source.subarray)

    calibration_index = DL1MonitoringEventIndexContainer()

    dl1_container = DL1ParametersContainer()

    if pointing_file_path:
        # Open drive report
        pointings = PointingPosition(drive_path=pointing_file_path)
        drive_data = pointings._read_drive_report()

    extra_im = ExtraImageInfo()
    extra_im.prefix = ''  # get rid of the prefix

    # get the first event to write array info and mc header
    event_iter = iter(source)
    first_event = next(event_iter)

    # Write extra information to the DL1 file
    write_array_info(subarray, output_filename)
    write_array_info_08(subarray, output_filename)

    if is_simu:
        write_mcheader(
            first_event.mcheader,
            output_filename,
            obs_id=first_event.index.obs_id,
            filters=filters,
            metadata=metadata,
        )

    with HDF5TableWriter(
            filename=output_filename,
            group_name='dl1/event',
            mode='a',
            filters=filters,
            add_prefix=True,
            # overwrite=True,
    ) as writer:

        if is_simu:
            subarray = subarray
            # build a mapping of tel_id back to tel_index:
            # (note this should be part of SubarrayDescription)
            idx = np.zeros(max(subarray.tel_indices) + 1)
            for key, val in subarray.tel_indices.items():
                idx[key] = val

            # the final transform then needs the mapping and the number of telescopes
            tel_list_transform = partial(
                utils.expand_tel_list,
                max_tels=max(subarray.tel) + 1,
            )

            writer.add_column_transform(table_name='subarray/trigger',
                                        col_name='tels_with_trigger',
                                        transform=tel_list_transform)

        # Forcing filters for the dl1 dataset that are currently read from the pre-existing files
        # This should be fixed in ctapipe and then corrected here
        writer._h5file.filters = filters
        logger.info(f"USING FILTERS: {writer._h5file.filters}")

        first_valid_ucts = None
        first_valid_ucts_tib = None
        previous_ucts_time_unix = []
        previous_ucts_trigger_type = []

        for i, event in enumerate(chain([first_event], event_iter)):

            if i % 100 == 0:
                logger.info(i)

            event.dl0.prefix = ''
            event.mc.prefix = 'mc'
            event.trigger.prefix = ''

            dl1_container.reset()

            # write sub tables
            if is_simu:
                write_subarray_tables(writer, event, metadata)
                if not custom_calibration:
                    cal_mc(event)
                if config['mc_image_scaling_factor'] != 1:
                    rescale_dl1_charge(event,
                                       config['mc_image_scaling_factor'])

            else:
                if i == 0:
                    # initialize the telescope
                    # FIXME? LST calibrator is only for one telescope
                    # it should be inside the telescope loop (?)

                    tel_id = calibration_calculator.tel_id

                    # write the first calibration event (initialized from calibration h5 file)
                    write_calibration_data(
                        writer,
                        calibration_index,
                        r1_dl1_calibrator.mon_data.tel[tel_id],
                        new_ped=True,
                        new_ff=True)

                # drs4 calibrations
                r0_r1_calibrator.calibrate(event)

                # process interleaved events (pedestals, ff, calibration)
                new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
                    event)

                # write monitoring containers if updated
                if new_ped_event or new_ff_event:
                    write_calibration_data(writer,
                                           calibration_index,
                                           event.mon.tel[tel_id],
                                           new_ped=new_ped_event,
                                           new_ff=new_ff_event)

                # calibrate and extract image from event
                r1_dl1_calibrator(event)

            # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
            # When the last version of the method is implemented, vol. reduction will be done at dl0
            apply_volume_reduction(event, subarray, config)

            # FIXME? This should be eventually done after we evaluate whether the image is
            # a candidate muon ring. In that case the full image could be kept, or reduced
            # only after the ring analysis is complete.

            for ii, telescope_id in enumerate(event.r0.tels_with_data):

                dl1_container.reset()

                # update the calibration index in the dl1 event container
                dl1_container.calibration_id = calibration_index.calibration_id

                dl1_container.fill_event_info(event)

                tel = event.dl1.tel[telescope_id]
                tel.prefix = ''  # don't really need one
                # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
                tel_name = str(subarray.tel[telescope_id])[4:]

                if custom_calibration:
                    lst_calibration(event, telescope_id)

                write_event = True
                # Will determine whether this event has to be written to the
                # DL1 output or not.

                if is_simu:
                    dl1_container.fill_mc(event,
                                          subarray.positions[telescope_id])

                try:
                    get_dl1(event,
                            subarray,
                            telescope_id,
                            dl1_container=dl1_container,
                            custom_config=config,
                            use_main_island=True)

                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')

                if not is_simu:
                    # GPS + WRS + UCTS is now working in its nominal configuration.
                    # These TS are stored into ucts_time container.
                    # TS can be alternatively calculated from the TIB and
                    # Dragon modules counters based on the first valid UCTS TS
                    # as the reference point. For the time being, the three TS
                    # are stored in the DL1 files for checking purposes.

                    module_id = 82  # Get counters from the central Dragon module

                    if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \
                            and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0):
                        # Dragon/TIB timestamps not based on a valid absolute reference timestamp

                        dragon_time = (event.lst.tel[telescope_id].svc.date +
                                       event.lst.tel[telescope_id].evt.
                                       pps_counter[module_id] +
                                       event.lst.tel[telescope_id].evt.
                                       tenMHz_counter[module_id] * 10**(-7))

                        tib_time = (
                            event.lst.tel[telescope_id].svc.date +
                            event.lst.tel[telescope_id].evt.tib_pps_counter +
                            event.lst.tel[telescope_id].evt.tib_tenMHz_counter
                            * 10**(-7))

                        if event.lst.tel[
                                telescope_id].evt.extdevices_presence & 2:
                            # UCTS presence flag is OK
                            ucts_time = event.lst.tel[
                                telescope_id].evt.ucts_timestamp * 1e-9  # secs

                            if first_valid_ucts is None:
                                first_valid_ucts = ucts_time

                                initial_dragon_counter = (
                                    event.lst.tel[telescope_id].evt.
                                    pps_counter[module_id] +
                                    event.lst.tel[telescope_id].evt.
                                    tenMHz_counter[module_id] * 10**(-7))
                                logger.warning(
                                    f"Dragon timestamps not based on a valid absolute reference timestamp. "
                                    f"Consider using the following initial values \n"
                                    f"Event ID: {event.index.event_id}, "
                                    f"First valid UCTS timestamp: {first_valid_ucts:.9f} s, "
                                    f"corresponding Dragon counter {initial_dragon_counter:.9f} s"
                                )

                            if event.lst.tel[telescope_id].evt.extdevices_presence & 1 \
                                    and first_valid_ucts_tib is None:
                                # Both TIB and UCTS presence flags are OK
                                first_valid_ucts_tib = ucts_time

                                initial_tib_counter = (
                                    event.lst.tel[telescope_id].evt.
                                    tib_pps_counter +
                                    event.lst.tel[telescope_id].evt.
                                    tib_tenMHz_counter * 10**(-7))
                                logger.warning(
                                    f"TIB timestamps not based on a valid absolute reference timestamp. "
                                    f"Consider using the following initial values \n"
                                    f"Event ID: {event.index.event_id}, UCTS timestamp corresponding to "
                                    f"the first valid TIB counter: {first_valid_ucts_tib:.9f} s, "
                                    f"corresponding TIB counter {initial_tib_counter:.9f} s"
                                )
                        else:
                            ucts_time = math.nan

                    else:
                        # Dragon/TIB timestamps based on a valid absolute reference UCTS timestamp
                        dragon_time = (
                            (ucts_t0_dragon - dragon_counter0) * 1e-9 +  # secs
                            event.lst.tel[telescope_id].evt.
                            pps_counter[module_id] +
                            event.lst.tel[telescope_id].evt.
                            tenMHz_counter[module_id] * 10**(-7))

                        tib_time = (
                            (ucts_t0_tib - tib_counter0) * 1e-9 +  # secs
                            event.lst.tel[telescope_id].evt.tib_pps_counter +
                            event.lst.tel[telescope_id].evt.tib_tenMHz_counter
                            * 10**(-7))

                        if event.lst.tel[
                                telescope_id].evt.extdevices_presence & 2:
                            # UCTS presence flag is OK
                            ucts_time = event.lst.tel[
                                telescope_id].evt.ucts_timestamp * 1e-9  # secs
                            if first_valid_ucts is None:
                                first_valid_ucts = ucts_time
                            if first_valid_ucts_tib is None \
                                    and event.lst.tel[telescope_id].evt.extdevices_presence & 1:
                                first_valid_ucts_tib = ucts_time
                        else:
                            ucts_time = math.nan

                    # FIXME: directly use unix_tai format whenever astropy v4.1 is out
                    ucts_time_utc = unix_tai_to_time(ucts_time)
                    dragon_time_utc = unix_tai_to_time(dragon_time)
                    tib_time_utc = unix_tai_to_time(tib_time)

                    dl1_container.ucts_time = ucts_time_utc.unix
                    dl1_container.dragon_time = dragon_time_utc.unix
                    dl1_container.tib_time = tib_time_utc.unix

                    # Until the TIB trigger_type is fully reliable, we also add
                    # the ucts_trigger_type to the data
                    dl1_container.ucts_trigger_type = event.lst.tel[
                        telescope_id].evt.ucts_trigger_type

                    # Due to a DAQ bug, sometimes there are 'jumps' in the
                    # UCTS info in the raw files. After one such jump,
                    # all the UCTS info attached to an event actually
                    # corresponds to the next event. This one-event
                    # shift stays like that until there is another jump
                    # (then it becomes a 2-event shift and so on). We will
                    # keep track of those jumps, by storing the UCTS info
                    # of the previously read events in the list
                    # previous_ucts_time_unix. The list has one element
                    # for each of the jumps, so if there has been just
                    # one jump we have the UCTS info of the previous
                    # event only (which truly corresponds to the
                    # current event). If there have been n jumps, we keep
                    # the past n events. The info to be used for
                    # the current event is always the first element of
                    # the array, previous_ucts_time_unix[0], whereas the
                    # current event's (wrong) ucts info is placed last in
                    # the array. Each time the first array element is
                    # used, it is removed and the rest move up in the
                    # list. We have another similar array for the trigger
                    # types, previous_ucts_trigger_type
                    #
                    if len(previous_ucts_time_unix) > 0:
                        # keep the time & trigger type read for this
                        # event (which really correspond to a later event):
                        current_ucts_time = dl1_container.ucts_time
                        current_ucts_trigger_type = dl1_container.ucts_trigger_type
                        # put in dl1_container the proper time for this
                        # event:
                        dl1_container.ucts_time = \
                            previous_ucts_time_unix.pop(0)
                        dl1_container.ucts_trigger_type = \
                            previous_ucts_trigger_type.pop(0)

                        # now put the current values last in the list,
                        # for later use:
                        previous_ucts_time_unix.append(current_ucts_time)
                        previous_ucts_trigger_type.\
                            append(current_ucts_trigger_type)

                    # Now check consistency of UCTS and Dragon times. If
                    # UCTS time is ahead of Dragon time by more than
                    # 1.e-6 s, most likely the UCTS info has been
                    # lost for this event (i.e. there has been another
                    # 'jump' of those described above), and the one we have
                    # actually corresponds to the next event. So we put it
                    # back first in the list, to assign it to the next
                    # event. We also move the other elements down in the
                    # list,  which will now be one element longer.
                    # We leave the current event with the same time,
                    # which will be approximately correct (depending on
                    # event rate), and set its ucts_trigger_type to -1,
                    # which will tell us a jump happened and hence this
                    # event does not have proper UCTS info.

                    if dl1_container.ucts_time - dl1_container.dragon_time > 1.e-6:
                        previous_ucts_time_unix.\
                            insert( 0, dl1_container.ucts_time)
                        previous_ucts_trigger_type.\
                            insert(0, dl1_container.ucts_trigger_type)
                        dl1_container.ucts_trigger_type = -1

                    # Select the timestamps to be used for pointing interpolation
                    if config['timestamps_pointing'] == "ucts":
                        event_timestamps = dl1_container.ucts_time
                    elif config['timestamps_pointing'] == "dragon":
                        event_timestamps = dragon_time_utc.unix
                    elif config['timestamps_pointing'] == "tib":
                        event_timestamps = tib_time_utc.unix
                    else:
                        raise ValueError(
                            "The timestamps_pointing option is not a valid one. \
                                         Try ucts (default), dragon or tib.")

                    if pointing_file_path and event_timestamps > 0:
                        azimuth, altitude = pointings.cal_pointingposition(
                            event_timestamps, drive_data)
                        event.pointing.tel[telescope_id].azimuth = azimuth
                        event.pointing.tel[telescope_id].altitude = altitude
                        dl1_container.az_tel = azimuth
                        dl1_container.alt_tel = altitude
                    else:
                        dl1_container.az_tel = u.Quantity(np.nan, u.rad)
                        dl1_container.alt_tel = u.Quantity(np.nan, u.rad)

                dl1_container.trigger_time = event.r0.tel[
                    telescope_id].trigger_time
                dl1_container.trigger_type = event.r0.tel[
                    telescope_id].trigger_type

                # FIXME: no need to read telescope characteristics like foclen for every event!
                foclen = subarray.tel[
                    telescope_id].optics.equivalent_focal_length
                mirror_area = u.Quantity(
                    subarray.tel[telescope_id].optics.mirror_area, u.m**2)
                dl1_container.prefix = tel.prefix

                # extra info for the image table
                extra_im.tel_id = telescope_id
                extra_im.selected_gain_channel = event.r1.tel[
                    telescope_id].selected_gain_channel

                for container in [extra_im, dl1_container, event.r0, tel]:
                    add_global_metadata(container, metadata)

                event.r0.prefix = ''

                writer.write(table_name=f'telescope/image/{tel_name}',
                             containers=[event.index, tel, extra_im])
                writer.write(table_name=f'telescope/parameters/{tel_name}',
                             containers=[event.index, dl1_container])

                # Muon ring analysis, for real data only (MC is done starting from DL1 files)
                if not is_simu:
                    bad_pixels = event.mon.tel[
                        telescope_id].calibration.unusable_pixels[0]
                    # Set to 0 unreliable pixels:
                    image = tel.image * (~bad_pixels)

                    # process only promising events, in terms of # of pixels with large signals:
                    if tag_pix_thr(image):

                        # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
                        numsamples = event.r1.tel[telescope_id].waveform.shape[
                            2]  # not necessarily the same as in r0!
                        bad_pixels_hg = event.mon.tel[
                            telescope_id].calibration.unusable_pixels[0]
                        bad_pixels_lg = event.mon.tel[
                            telescope_id].calibration.unusable_pixels[1]
                        # Now set to 0 all samples in unreliable pixels. Important for global peak
                        # integrator in case of crazy pixels!  TBD: can this be done in a simpler
                        # way?
                        bad_waveform = np.array(
                            ([
                                np.transpose(
                                    np.array(numsamples * [bad_pixels_hg])),
                                np.transpose(
                                    np.array(numsamples * [bad_pixels_lg]))
                            ]))

                        # print('hg bad pixels:',np.where(bad_pixels_hg))
                        # print('lg bad pixels:',np.where(bad_pixels_lg))

                        event.r1.tel[telescope_id].waveform *= ~bad_waveform
                        r1_dl1_calibrator_for_muon_rings(event)

                        tel = event.dl1.tel[telescope_id]
                        image = tel.image * (~bad_pixels)

                        # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
                        # perhaps the event is no longer promising (e.g. if it has a large time evolution)
                        if not tag_pix_thr(image):
                            good_ring = False
                        else:
                            # read geometry from event.inst. But not needed for every event. FIXME?
                            geom = subarray.tel[telescope_id].\
                                camera.geometry

                            muonintensityparam, dist_mask, \
                            ring_size, size_outside_ring, muonringparam, \
                            good_ring, radial_distribution, \
                            mean_pixel_charge_around_ring,\
                            muonpars = \
                                analyze_muon_event(subarray,
                                                   event.index.event_id,
                                                   image, geom, foclen,
                                                   mirror_area, False, '')
                            #                      mirror_area, True, './')
                            #           (test) plot muon rings as png files

                            # Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks:
                            bright_pixels_waveforms = event.r1.tel[
                                telescope_id].waveform[:, image >
                                                       min_pe_for_muon_t_calc, :]
                            stacked_waveforms = np.sum(bright_pixels_waveforms,
                                                       axis=-2)
                            # stacked waveforms from all bright pixels; shape (ngains, nsamples)
                            hg_peak_sample = np.argmax(stacked_waveforms,
                                                       axis=-1)[0]
                            lg_peak_sample = np.argmax(stacked_waveforms,
                                                       axis=-1)[1]

                        if good_ring:
                            fill_muon_event(-1, muon_parameters, good_ring,
                                            event.index.event_id, dragon_time,
                                            muonintensityparam, dist_mask,
                                            muonringparam, radial_distribution,
                                            ring_size, size_outside_ring,
                                            mean_pixel_charge_around_ring,
                                            muonpars, hg_peak_sample,
                                            lg_peak_sample)

                # writes mc information per telescope, including photo electron image
                if is_simu \
                        and (event.mc.tel[telescope_id].true_image > 0).any() \
                        and config['write_pe_image']:
                    event.mc.tel[telescope_id].prefix = ''
                    writer.write(
                        table_name=f'simulation/{tel_name}',
                        containers=[event.mc.tel[telescope_id], extra_im])

        if not is_simu:
            # at the end of event loop ask calculation of remaining interleaved statistics
            new_ped, new_ff = calibration_calculator.output_interleaved_results(
                event)
            # write monitoring events
            write_calibration_data(writer,
                                   calibration_index,
                                   event.mon.tel[tel_id],
                                   new_ped=new_ped,
                                   new_ff=new_ff)

    if first_valid_ucts is None:
        logger.warning("Not valid UCTS timestamp found")

    if first_valid_ucts_tib is None:
        logger.warning("Not valid TIB counter value found")

    if is_simu:
        # Reconstruct source position from disp for all events and write the result in the output file
        for tel_name in ['LST_LSTCam']:
            focal = OpticsDescription.from_name(
                tel_name.split('_')[0]).equivalent_focal_length
            add_disp_to_parameters_table(output_filename,
                                         dl1_params_lstcam_key, focal)

        # Write energy histogram from simtel file and extra metadata
        # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
        if source.max_events is None:
            write_simtel_energy_histogram(source,
                                          output_filename,
                                          obs_id=event.index.obs_id,
                                          metadata=metadata)
    else:
        dir, name = os.path.split(output_filename)
        name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
        # Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
        name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
        muon_output_filename = Path(dir, name)
        table = Table(muon_parameters)
        table.write(muon_output_filename, format='fits', overwrite=True)

        # Produce the dl1 datacheck .h5 file:
        check_dl1(output_filename,
                  Path(output_filename).parent,
                  max_cores=1,
                  create_pdf=False)
Exemplo n.º 19
0
def get_events(filename,
               storedata=False,
               test=False,
               concatenate=False,
               storeimg=False,
               outdir='./results/'):
    """
    Depreciated, use r0_to_dl1.

    Read a Simtelarray file, extract pixels charge, calculate image
    parameters and timing parameters and store the result in an hdf5
    file.

    Parameters:
    -----------
    filename: str
    Name of the simtelarray file.

    storedata: boolean
    True: store extracted data in a hdf5 file

    concatenate: boolean
    True: store the extracted data at the end of an existing file

    storeimg: boolean
    True: store also pixel data

    outdir: srt
    Output directory

    Returns:
    --------
    pandas DataFrame: output
    """
    from warnings import warn
    warn("Deprecated: use r0_to_dl1")

    #Particle type:
    particle_type = utils.guess_type(filename)

    #Create data frame where DL2 data will be stored:

    features = [
        'obs_id',
        'event_id',
        'mc_energy',
        'mc_alt',
        'mc_az',
        'mc_core_x',
        'mc_core_y',
        'mc_h_first_int',
        'mc_type',
        'gps_time',
        'width',
        'length',
        'wl',
        'phi',
        'psi',
        'r',
        'x',
        'y',
        'intensity',
        'skewness',
        'kurtosis',
        'mc_alt_tel',
        'mc_az_tel',
        'mc_core_distance',
        'mc_x_max',
        'time_gradient',
        'intercept',
        'src_x',
        'src_y',
        'disp_norm',
    ]

    output = pd.DataFrame(columns=features)

    #Read LST1 events:
    source = event_source(input_url=filename,
                          allowed_tels={1})  #Open Simtelarray file

    #Cleaning levels:

    level1 = {'LSTCam': 6.}
    level2 = level1.copy()
    # We use as second cleaning level just half of the first cleaning level

    for key in level2:
        level2[key] *= 0.5

    log10pixelHGsignal = {}
    survived = {}

    imagedata = np.array([])

    for key in level1:

        log10pixelHGsignal[key] = []
        survived[key] = []
    i = 0
    for event in source:
        if i % 100 == 0:
            print("EVENT_ID: ", event.r0.event_id, "TELS: ",
                  event.r0.tels_with_data, "MC Energy:", event.mc.energy)

        i = i + 1

        ntels = len(event.r0.tels_with_data)

        if test == True and i > 1000:  # for quick tests
            break

        for ii, tel_id in enumerate(event.r0.tels_with_data):

            geom = event.inst.subarray.tel[tel_id].camera  #Camera geometry
            tel_coords = event.inst.subarray.tel_coords[
                event.inst.subarray.tel_indices[tel_id]]

            data = event.r0.tel[tel_id].waveform

            ped = event.mc.tel[tel_id].pedestal  # the pedestal is the
            #average (for pedestal events) of the *sum* of all samples,
            #from sim_telarray

            nsamples = data.shape[2]  # total number of samples

            # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix

            pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples

            integrator = LocalPeakWindowSum()
            integration, pulse_time = integrator(
                pedcorrectedsamples
            )  # these are 2D matrices num_gains * num_pixels

            chan = 0  # high gain used for now...
            signals = integration[chan].astype(float)

            dc2pe = event.mc.tel[tel_id].dc_to_pe  # numgains * numpixels
            signals *= dc2pe[chan]

            # Add all individual pixel signals to the numpy array of the
            # corresponding camera inside the log10pixelsignal dictionary

            log10pixelHGsignal[str(geom)].extend(np.log10(signals))

            # Apply image cleaning

            cleanmask = tailcuts_clean(geom,
                                       signals,
                                       picture_thresh=level1[str(geom)],
                                       boundary_thresh=level2[str(geom)],
                                       keep_isolated_pixels=False,
                                       min_number_picture_neighbors=1)

            survived[str(geom)].extend(cleanmask)

            clean = signals.copy()
            clean[~cleanmask] = 0.0  # set to 0 pixels which did not
            # survive cleaning

            if np.max(clean) < 1.e-6:  # skip images with no pixels
                continue

            # Calculate image parameters

            hillas = hillas_parameters(geom, clean)
            foclen = event.inst.subarray.tel[
                tel_id].optics.equivalent_focal_length

            w = np.rad2deg(np.arctan2(hillas.width, foclen))
            l = np.rad2deg(np.arctan2(hillas.length, foclen))

            #Calculate Timing parameters

            peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns")
            timepars = time.timing_parameters(geom, clean, peak_time, hillas)

            if w >= 0:
                if storeimg == True:
                    if imagedata.size == 0:
                        imagedata = clean
                    else:
                        imagedata = np.vstack([imagedata,
                                               clean])  #Pixel content

                #Hillas parameters
                width = w.value
                length = l.value
                phi = hillas.phi.value
                psi = hillas.psi.value
                r = hillas.r.value
                x = hillas.x.value
                y = hillas.y.value
                intensity = np.log10(hillas.intensity)
                skewness = hillas.skewness
                kurtosis = hillas.kurtosis

                #MC data:
                obs_id = event.r0.obs_id
                event_id = event.r0.event_id

                mc_energy = np.log10(event.mc.energy.value *
                                     1e3)  #Log10(Energy) in GeV
                mc_alt = event.mc.alt.value
                mc_az = event.mc.az.value
                mc_core_x = event.mc.core_x.value
                mc_core_y = event.mc.core_y.value
                mc_h_first_int = event.mc.h_first_int.value
                mc_type = event.mc.shower_primary_id
                mc_az_tel = event.mcheader.run_array_direction[0].value
                mc_alt_tel = event.mcheader.run_array_direction[1].value
                mc_x_max = event.mc.x_max.value
                gps_time = event.trig.gps_time.value

                #Calculate mc_core_distance parameters

                mc_core_distance = np.sqrt(
                    (tel_coords.x.value - event.mc.core_x.value)**2 +
                    (tel_coords.y.value - event.mc.core_y.value)**2)

                #Timing parameters

                time_gradient = timepars['slope'].value
                intercept = timepars['intercept']

                #Calculate disp_ and Source position in camera coordinates

                tel = OpticsDescription.from_name(
                    'LST')  #Telescope description
                focal_length = tel.equivalent_focal_length.value
                sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel,
                                                     mc_az_tel, focal_length)
                src_x = sourcepos[0]
                src_y = sourcepos[1]
                disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y)

                eventdf = pd.DataFrame([[
                    obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x,
                    mc_core_y, mc_h_first_int, mc_type, gps_time, width,
                    length, width / length, phi, psi, r, x, y, intensity,
                    skewness, kurtosis, mc_alt_tel, mc_az_tel,
                    mc_core_distance, mc_x_max, time_gradient, intercept,
                    src_x, src_y, disp, mc_type
                ]],
                                       columns=features)

                output = output.append(eventdf, ignore_index=True)

    outfile = outdir + particle_type + '_events.hdf5'

    if storedata == True:
        if (concatenate == False
                or (concatenate == True
                    and np.DataSource().exists(outfile) == False)):
            output.to_hdf(outfile, key=particle_type + "_events", mode="w")
            if storeimg == True:
                f = h5py.File(outfile, 'r+')
                f.create_dataset('images', data=imagedata)
                f.close()
        else:
            if storeimg == True:
                f = h5py.File(outfile, 'r')
                images = f['images']
                del f['images']
                images = np.vstack([images, imagedata])
                f.close()
                saved = pd.read_hdf(outfile, key=particle_type + '_events')
                output = saved.append(output, ignore_index=True)
                output.to_hdf(outfile, key=particle_type + "_events", mode="w")
                f = h5py.File(outfile, 'r+')
                f.create_dataset('images', data=images)
                f.close()
            else:
                saved = pd.read_hdf(outfile, key=particle_type + '_events')
                output = saved.append(output, ignore_index=True)
                output.to_hdf(outfile, key=particle_type + "_events", mode="w")
    del source
    return output
Exemplo n.º 20
0
def get_events(filename,
               storedata=False,
               concatenate=False,
               storeimg=False,
               outdir='./results/'):
    """
    Read a Simtelarray file, extract pixels charge, calculate image parameters and timing
    parameters and store the result in an hdf5 file. 
    
    Parameters:
    filename: str
    Name of the simtelarray file.

    storedata: boolean
    True: store extracted data in a hdf5 file

    concatenate: boolean
    True: store the extracted data at the end of an existing file

    storeimg: boolean
    True: store also pixel data
    
    outdir: srt
    Output directory
    
    
    
    """
    #Particle type:

    particle_type = guess_type(filename)

    #Create data frame where DL2 data will be stored:

    features = [
        'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y',
        'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi',
        'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel',
        'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX',
        'SrcY', 'disp', 'hadroness'
    ]
    output = pd.DataFrame(columns=features)

    #Read LST1 events:
    source = EventSourceFactory.produce(
        input_url=filename, allowed_tels={1})  #Open Simtelarray file

    #Cleaning levels:

    level1 = {'LSTCam': 6.}
    level2 = level1.copy()
    # We use as second cleaning level just half of the first cleaning level
    for key in level2:
        level2[key] *= 0.5

    log10pixelHGsignal = {}
    survived = {}

    imagedata = np.array([])

    for key in level1:

        log10pixelHGsignal[key] = []
        survived[key] = []
    i = 0
    for event in source:
        if i % 100 == 0:
            print("EVENT_ID: ", event.r0.event_id, "TELS: ",
                  event.r0.tels_with_data, "MC Energy:", event.mc.energy)
        i = i + 1
        ntels = len(event.r0.tels_with_data)
        '''
        if i > 100:   # for quick tests
            break
        '''
        for ii, tel_id in enumerate(event.r0.tels_with_data):

            geom = event.inst.subarray.tel[tel_id].camera  #Camera geometry
            tel_coords = event.inst.subarray.tel_coords[
                event.inst.subarray.tel_indices[tel_id]]
            data = event.r0.tel[tel_id].waveform
            ped = event.mc.tel[tel_id].pedestal
            # the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray

            nsamples = data.shape[2]  # total number of samples
            pedcorrectedsamples = data - np.atleast_3d(
                ped
            ) / nsamples  # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix

            integrator = LocalPeakIntegrator(None, None)
            integration, peakpos, window = integrator.extract_charge(
                pedcorrectedsamples
            )  # these are 2D matrices num_gains * num_pixels

            chan = 0  # high gain used for now...
            signals = integration[chan].astype(float)

            dc2pe = event.mc.tel[tel_id].dc_to_pe  # numgains * numpixels
            signals *= dc2pe[chan]

            # Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary
            log10pixelHGsignal[str(geom)].extend(
                np.log10(signals)
            )  # This seems to be faster like this, with normal python lists

            # Apply image cleaning
            cleanmask = tailcuts_clean(geom,
                                       signals,
                                       picture_thresh=level1[str(geom)],
                                       boundary_thresh=level2[str(geom)],
                                       keep_isolated_pixels=False,
                                       min_number_picture_neighbors=1)
            survived[str(geom)].extend(
                cleanmask
            )  # This seems to be faster like this, with normal python lists

            clean = signals.copy()
            clean[
                ~cleanmask] = 0.0  # set to 0 pixels which did not survive cleaning
            if np.max(clean) < 1.e-6:  # skip images with no pixels
                continue

            # Calculate image parameters
            hillas = hillas_parameters(
                geom,
                clean)  # this one gives some warnings invalid value in sqrt
            foclen = event.inst.subarray.tel[
                tel_id].optics.equivalent_focal_length

            w = np.rad2deg(np.arctan2(hillas.width, foclen))
            l = np.rad2deg(np.arctan2(hillas.length, foclen))

            #Calculate Timing parameters

            peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns")
            timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean,
                                              peak_time, hillas.psi)

            if w >= 0:

                if storeimg == True:
                    if imagedata.size == 0:
                        imagedata = clean
                    else:
                        imagedata = np.vstack([imagedata,
                                               clean])  #Pixel content

                width = w.value
                length = l.value
                phi = hillas.phi.value
                psi = hillas.psi.value
                r = hillas.r.value
                x = hillas.x.value
                y = hillas.y.value
                intensity = np.log10(hillas.intensity)
                skewness = hillas.skewness
                kurtosis = hillas.kurtosis

                #Store parameters from event and MC:
                ObsID = event.r0.obs_id
                EvID = event.r0.event_id

                mcEnergy = np.log10(event.mc.energy.value *
                                    1e3)  #Log10(Energy) in GeV
                mcAlt = event.mc.alt.value
                mcAz = event.mc.az.value
                mcCore_x = event.mc.core_x.value
                mcCore_y = event.mc.core_y.value
                mcHfirst = event.mc.h_first_int.value
                mcType = event.mc.shower_primary_id
                mcAztel = event.mcheader.run_array_direction[0].value
                mcAlttel = event.mcheader.run_array_direction[1].value
                mcXmax = event.mc.x_max.value
                GPStime = event.trig.gps_time.value

                impact = np.sqrt(
                    (tel_coords.x.value - event.mc.core_x.value)**2 +
                    (tel_coords.y.value - event.mc.core_y.value)**2)

                time_gradient = timepars[0].value
                intercept = timepars[1].value

                #Calculate Disp and Source position in camera coordinates
                tel = OpticsDescription.from_name(
                    'LST')  #Telescope description
                focal_length = tel.equivalent_focal_length.value
                sourcepos = transformations.calc_CamSourcePos(
                    mcAlt, mcAz, mcAlttel, mcAztel, focal_length)
                SrcX = sourcepos[0]
                SrcY = sourcepos[1]
                disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x,
                                                 y)

                hadroness = 0
                if particle_type == 'proton':
                    hadroness = 1

                eventdf = pd.DataFrame([[
                    ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y,
                    mcHfirst, mcType, GPStime, width, length, width / length,
                    phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel,
                    mcAztel, impact, mcXmax, time_gradient, intercept, SrcX,
                    SrcY, disp, hadroness
                ]],
                                       columns=features)

                output = output.append(eventdf, ignore_index=True)

    outfile = outdir + particle_type + '_events.hdf5'

    if storedata == True:

        if concatenate == False or (concatenate == True and
                                    np.DataSource().exists(outfile) == False):
            output.to_hdf(outfile, key=particle_type + "_events", mode="w")
            if storeimg == True:
                f = h5py.File(outfile, 'r+')
                f.create_dataset('images', data=imagedata)
                f.close()
        else:
            if storeimg == True:
                f = h5py.File(outfile, 'r')
                images = f['images']
                del f['images']
                images = np.vstack([images, imagedata])
                f.close()
                saved = pd.read_hdf(outfile, key=particle_type + '_events')
                output = saved.append(output, ignore_index=True)
                output.to_hdf(outfile, key=particle_type + "_events", mode="w")
                f = h5py.File(outfile, 'r+')
                f.create_dataset('images', data=images)
                f.close()
            else:
                saved = pd.read_hdf(outfile, key=particle_type + '_events')
                output = saved.append(output, ignore_index=True)
                output.to_hdf(outfile, key=particle_type + "_events", mode="w")
    del source
    return output
Exemplo n.º 21
0
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
              output_filename=None,
              custom_config={},
              pedestal_path=None,
              calibration_path=None,
              time_calibration_path=None,
              pointing_file_path=None,
              ucts_t0_dragon=math.nan,
              dragon_counter0=math.nan,
              ucts_t0_tib=math.nan,
              tib_counter0=math.nan):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str
        path to output file, default: `./` + basename(input_filename)
    custom_config: path to a configuration file
    pedestal_path: Path to the DRS4 pedestal file
    calibration_path: Path to the file with calibration constants and
        pedestals
    time_calibration_path: Path to the DRS4 time correction file
    pointing_file_path: path to the Drive log with the pointing information
    Arguments below are just temporal and will be removed whenever UCTS+EvB
    is proved to stably and reliably provide timestamps.
    ucts_t0_dragon: first valid ucts_time
    dragon_counter0: Dragon counter corresponding to ucts_t0_dragon
    ucts_t0_tib: first valid ucts_time for the first valid TIB counter
    tib_counter0: first valid TIB counter

    Returns
    -------

    """
    if output_filename is None:
        output_filename = ('dl1_' +
                           os.path.basename(input_filename).rsplit('.', 1)[0] +
                           '.h5')
    if os.path.exists(output_filename):
        raise AttributeError(output_filename + ' exists, exiting.')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]

    try:
        source = event_source(input_filename, back_seekable=True)
    except:
        # back_seekable might not be available for other sources that eventio
        # TODO for real data: source with calibration file and pointing file
        source = event_source(input_filename)

    is_simu = source.metadata['is_simulation']

    source.allowed_tels = config["allowed_tels"]
    if config["max_events"] is not None:
        source.max_events = config["max_events"] + 1

    metadata = global_metadata(source)
    write_metadata(metadata, output_filename)

    cal_mc = load_calibrator_from_config(config)

    # minimum number of pe in a pixel to include it in calculation of muon ring time (peak sample):
    min_pe_for_muon_t_calc = 10.

    # Dictionary to store muon ring parameters
    muon_parameters = create_muon_table()

    if not is_simu:

        # TODO : add DRS4 calibration config in config file, read it and pass it here
        r0_r1_calibrator = LSTR0Corrections(pedestal_path=pedestal_path,
                                            tel_id=1)

        # all this will be cleaned up in a next PR related to the configuration files
        r1_dl1_calibrator = LSTCameraCalibrator(
            calibration_path=calibration_path,
            time_calibration_path=time_calibration_path,
            extractor_product=config['image_extractor'],
            gain_threshold=Config(config).gain_selector_config['threshold'],
            config=Config(config),
            allowed_tels=[1],
        )

        # Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
        # using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
        r1_dl1_calibrator_for_muon_rings = LSTCameraCalibrator(
            calibration_path=calibration_path,
            time_calibration_path=time_calibration_path,
            extractor_product=config['image_extractor_for_muons'],
            gain_threshold=Config(config).gain_selector_config['threshold'],
            config=Config(config),
            allowed_tels=[1],
        )

    dl1_container = DL1ParametersContainer()

    if pointing_file_path:
        # Open drive report
        pointings = PointingPosition()
        pointings.drive_path = pointing_file_path
        drive_data = pointings._read_drive_report()

    extra_im = ExtraImageInfo()
    extra_im.prefix = ''  # get rid of the prefix

    event = next(iter(source))

    write_array_info(event, output_filename)
    ### Write extra information to the DL1 file
    if is_simu:
        write_mcheader(event.mcheader,
                       output_filename,
                       obs_id=event.r0.obs_id,
                       filters=filters,
                       metadata=metadata)
        subarray = event.inst.subarray

    with HDF5TableWriter(
            filename=output_filename,
            group_name='dl1/event',
            mode='a',
            filters=filters,
            add_prefix=True,
            # overwrite = True,
    ) as writer:

        print("USING FILTERS: ", writer._h5file.filters)

        if is_simu:
            # build a mapping of tel_id back to tel_index:
            # (note this should be part of SubarrayDescription)
            idx = np.zeros(max(subarray.tel_indices) + 1)
            for key, val in subarray.tel_indices.items():
                idx[key] = val

            # the final transform then needs the mapping and the number of telescopes
            tel_list_transform = partial(
                utils.expand_tel_list,
                max_tels=len(event.inst.subarray.tel) + 1,
            )

            writer.add_column_transform(table_name='subarray/trigger',
                                        col_name='tels_with_trigger',
                                        transform=tel_list_transform)

        ### EVENT LOOP ###
        for i, event in enumerate(source):
            if i % 100 == 0:
                print(i)

            event.dl0.prefix = ''
            event.mc.prefix = 'mc'
            event.trig.prefix = ''

            # write sub tables
            if is_simu:
                write_subarray_tables(writer, event, metadata)
                if not custom_calibration:
                    cal_mc(event)

            else:
                r0_r1_calibrator.calibrate(event)
                r1_dl1_calibrator(event)

            # Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
            # When the last version of the method is implemented, vol. reduction will be done at dl0
            check_and_apply_volume_reduction(event, config)
            # FIXME? This should be eventually done after we evaluate whether the image is
            # a candidate muon ring. In that case the full image could be kept, or reduced
            # only after the ring analysis is complete.

            for ii, telescope_id in enumerate(event.r0.tels_with_data):

                tel = event.dl1.tel[telescope_id]
                tel.prefix = ''  # don't really need one
                # remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
                tel_name = str(event.inst.subarray.tel[telescope_id])[4:]
                tel_name = tel_name.replace('-003', '')

                if custom_calibration:
                    lst_calibration(event, telescope_id)

                try:
                    dl1_filled = get_dl1(event,
                                         telescope_id,
                                         dl1_container=dl1_container,
                                         custom_config=config,
                                         use_main_island=True)

                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')
                    continue

                if dl1_filled is not None:

                    # Some custom def
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    # Log10(Energy) in GeV
                    if is_simu:
                        dl1_container.mc_energy = event.mc.energy.value
                        dl1_container.log_mc_energy = np.log10(
                            event.mc.energy.value * 1e3)
                        dl1_container.fill_mc(event)

                    dl1_container.log_intensity = np.log10(
                        dl1_container.intensity)
                    dl1_container.gps_time = event.trig.gps_time.value

                    if not is_simu:
                        # GPS + WRS + UCTS is now working in its nominal configuration.
                        # These TS are stored into ucts_time container.
                        # TS can be alternatively calculated from the TIB and
                        # Dragon modules counters based on the first valid UCTS TS
                        # as the reference point. For the time being, the three TS
                        # are stored in the DL1 files for checking purposes.

                        ucts_time = event.lst.tel[
                            telescope_id].evt.ucts_timestamp * 1e-9  # secs

                        module_id = 82  # Get counters from the central Dragon module

                        if math.isnan(ucts_t0_dragon) and math.isnan(dragon_counter0) \
                           and math.isnan(ucts_t0_tib) and math.isnan(tib_counter0):
                            # Dragon/TIB timestamps not based on a valid absolute reference timestamp
                            dragon_time = (
                                event.lst.tel[telescope_id].svc.date +
                                event.lst.tel[telescope_id].evt.
                                pps_counter[module_id] +
                                event.lst.tel[telescope_id].evt.
                                tenMHz_counter[module_id] * 10**(-7))

                            tib_time = (
                                event.lst.tel[telescope_id].svc.date +
                                event.lst.tel[telescope_id].evt.tib_pps_counter
                                + event.lst.tel[telescope_id].evt.
                                tib_tenMHz_counter * 10**(-7))

                        else:
                            # Dragon/TIB timestamps based on a valid absolute reference timestamp
                            dragon_time = (
                                (ucts_t0_dragon - dragon_counter0) * 1e-9
                                +  # secs
                                event.lst.tel[telescope_id].evt.
                                pps_counter[module_id] +
                                event.lst.tel[telescope_id].evt.
                                tenMHz_counter[module_id] * 10**(-7))

                            tib_time = (
                                (ucts_t0_tib - tib_counter0) * 1e-9 +  # secs
                                event.lst.tel[telescope_id].evt.tib_pps_counter
                                + event.lst.tel[telescope_id].evt.
                                tib_tenMHz_counter * 10**(-7))

                        # FIXME: directly use unix_tai format whenever astropy v4.1 is out
                        ucts_time_utc = unix_tai_to_utc(ucts_time)
                        dragon_time_utc = unix_tai_to_utc(dragon_time)
                        tib_time_utc = unix_tai_to_utc(tib_time)

                        dl1_container.ucts_time = ucts_time_utc.unix
                        dl1_container.dragon_time = dragon_time_utc.unix
                        dl1_container.tib_time = tib_time_utc.unix

                        # Select the timestamps to be used for pointing interpolation
                        if config['timestamps_pointing'] == "ucts":
                            event_timestamps = ucts_time_utc.unix
                        elif config['timestamps_pointing'] == "dragon":
                            event_timestamps = dragon_time_utc.unix
                        elif config['timestamps_pointing'] == "tib":
                            event_timestamps = tib_time_utc.unix
                        else:
                            raise ValueError(
                                "The timestamps_pointing option is not a valid one. \
                                             Try ucts (default), dragon or tib."
                            )

                        if pointing_file_path and event_timestamps > 0:
                            azimuth, altitude = pointings.cal_pointingposition(
                                event_timestamps, drive_data)
                            event.pointing[telescope_id].azimuth = azimuth
                            event.pointing[telescope_id].altitude = altitude
                            dl1_container.az_tel = azimuth
                            dl1_container.alt_tel = altitude
                        else:
                            dl1_container.az_tel = u.Quantity(np.nan, u.rad)
                            dl1_container.alt_tel = u.Quantity(np.nan, u.rad)

                        # Until the TIB trigger_type is fully reliable, we also add
                        # the ucts_trigger_type to the data
                        extra_im.ucts_trigger_type = event.lst.tel[
                            telescope_id].evt.ucts_trigger_type

                    # FIXME: no need to read telescope characteristics like foclen for every event!
                    foclen = event.inst.subarray.tel[
                        telescope_id].optics.equivalent_focal_length
                    mirror_area = u.Quantity(
                        event.inst.subarray.tel[telescope_id].optics.
                        mirror_area, u.m**2)
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(
                        np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width.value
                    dl1_container.length = length.value

                    dl1_container.prefix = tel.prefix

                    extra_im.tel_id = telescope_id
                    extra_im.num_trig_pix = event.r0.tel[
                        telescope_id].num_trig_pix
                    extra_im.trigger_time = event.r0.tel[
                        telescope_id].trigger_time
                    extra_im.trigger_type = event.r0.tel[
                        telescope_id].trigger_type
                    extra_im.trig_pix_id = event.r0.tel[
                        telescope_id].trig_pix_id

                    for container in [extra_im, dl1_container, event.r0, tel]:
                        add_global_metadata(container, metadata)

                    event.r0.prefix = ''

                    writer.write(table_name=f'telescope/image/{tel_name}',
                                 containers=[event.r0, tel, extra_im])
                    writer.write(table_name=f'telescope/parameters/{tel_name}',
                                 containers=[dl1_container, extra_im])

                    # Muon ring analysis, for real data only (MC is done starting from DL1 files)
                    if not is_simu:
                        bad_pixels = event.mon.tel[
                            telescope_id].calibration.unusable_pixels[0]
                        # Set to 0 unreliable pixels:
                        image = tel.image * (~bad_pixels)

                        # process only promising events, in terms of # of pixels with large signals:
                        if tag_pix_thr(image):

                            # re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
                            numsamples = event.r1.tel[
                                telescope_id].waveform.shape[
                                    2]  # not necessarily the same as in r0!
                            bad_pixels_hg = event.mon.tel[
                                telescope_id].calibration.unusable_pixels[0]
                            bad_pixels_lg = event.mon.tel[
                                telescope_id].calibration.unusable_pixels[1]
                            # Now set to 0 all samples in unreliable pixels. Important for global peak
                            # integrator in case of crazy pixels!  TBD: can this be done in a simpler
                            # way?
                            bad_waveform = np.array(([
                                np.transpose(
                                    np.array(numsamples * [bad_pixels_hg])),
                                np.transpose(
                                    np.array(numsamples * [bad_pixels_lg]))
                            ]))

                            # print('hg bad pixels:',np.where(bad_pixels_hg))
                            # print('lg bad pixels:',np.where(bad_pixels_lg))

                            event.r1.tel[
                                telescope_id].waveform *= ~bad_waveform
                            r1_dl1_calibrator_for_muon_rings(event)

                            tel = event.dl1.tel[telescope_id]
                            image = tel.image * (~bad_pixels)

                            # Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
                            # perhaps the event is no longer promising (e.g. if it has a large time evolution)
                            if not tag_pix_thr(image):
                                good_ring = False
                            else:
                                # read geometry from event.inst. But not needed for every event. FIXME?
                                geom = event.inst.subarray.tel[
                                    telescope_id].camera

                                muonintensityparam, size_outside_ring, muonringparam, good_ring, \
                                    radial_distribution, mean_pixel_charge_around_ring = \
                                    analyze_muon_event(event.r0.event_id, image, geom, foclen,
                                                       mirror_area, False, '')
                                #                      mirror_area, True, './') # (test) plot muon rings as png files

                                # Now we want to obtain the waveform sample (in HG and LG) at which the ring light peaks:
                                bright_pixels_waveforms = event.r1.tel[
                                    telescope_id].waveform[:, image >
                                                           min_pe_for_muon_t_calc, :]
                                stacked_waveforms = np.sum(
                                    bright_pixels_waveforms, axis=-2)
                                # stacked waveforms from all bright pixels; shape (ngains, nsamples)
                                hg_peak_sample = np.argmax(stacked_waveforms,
                                                           axis=-1)[0]
                                lg_peak_sample = np.argmax(stacked_waveforms,
                                                           axis=-1)[1]

                            if good_ring:
                                fill_muon_event(muon_parameters, good_ring,
                                                event.r0.event_id, dragon_time,
                                                muonintensityparam,
                                                muonringparam,
                                                radial_distribution,
                                                size_outside_ring,
                                                mean_pixel_charge_around_ring,
                                                hg_peak_sample, lg_peak_sample)

                    # writes mc information per telescope, including photo electron image
                    if is_simu \
                            and (event.mc.tel[telescope_id].photo_electron_image > 0).any() \
                            and config['write_pe_image']:
                        event.mc.tel[telescope_id].prefix = ''
                        writer.write(
                            table_name=f'simulation/{tel_name}',
                            containers=[event.mc.tel[telescope_id], extra_im])

    if is_simu:
        ### Reconstruct source position from disp for all events and write the result in the output file
        for tel_name in ['LST_LSTCam']:
            focal = OpticsDescription.from_name(
                tel_name.split('_')[0]).equivalent_focal_length
            dl1_params_key = f'dl1/event/telescope/parameters/{tel_name}'
            add_disp_to_parameters_table(output_filename, dl1_params_key,
                                         focal)

        # Write energy histogram from simtel file and extra metadata
        # ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
        if source.max_events is None:
            write_simtel_energy_histogram(source,
                                          output_filename,
                                          obs_id=event.dl0.obs_id,
                                          metadata=metadata)
    else:
        dir = os.path.dirname(output_filename)
        name = os.path.basename(output_filename)
        k = name.find('Run')
        muon_output_filename = name[0:name.find('LST-')+5] + '.' + \
                               name[k:k+13] + '.fits'

        muon_output_filename = dir + '/' + muon_output_filename.replace(
            "dl1", "muons")
        table = Table(muon_parameters)
        table.write(muon_output_filename, format='fits', overwrite=True)
def main():
    std_config = get_standard_config()

    if args.config_file is not None:
        config = replace_config(std_config,
                                read_configuration_file(args.config_file))
    else:
        config = std_config

    print(config['tailcut'])

    foclen = OpticsDescription.from_name('LST').equivalent_focal_length
    cam_table = Table.read(args.input_file,
                           path="instrument/telescope/camera/LSTCam")
    camera_geom = CameraGeometry.from_table(cam_table)

    dl1_container = DL1ParametersContainer()
    parameters_to_update = list(HillasParametersContainer().keys())
    parameters_to_update.extend([
        'concentration_cog',
        'concentration_core',
        'concentration_pixel',
        'leakage_intensity_width_1',
        'leakage_intensity_width_2',
        'leakage_pixels_width_1',
        'leakage_pixels_width_2',
        'n_islands',
        'intercept',
        'time_gradient',
        'n_pixels',
        'wl',
        'r',
    ])

    nodes_keys = get_dataset_keys(args.input_file)
    if args.noimage:
        nodes_keys.remove(dl1_images_lstcam_key)

    auto_merge_h5files([args.input_file],
                       args.output_file,
                       nodes_keys=nodes_keys)

    with tables.open_file(args.input_file, mode='r') as input:
        image_table = input.root[dl1_images_lstcam_key]
        with tables.open_file(args.output_file, mode='a') as output:

            params = output.root[dl1_params_lstcam_key].read()

            for ii, row in enumerate(image_table):
                if ii % 10000 == 0:
                    print(ii)
                image = row['image']
                peak_time = row['peak_time']

                signal_pixels = tailcuts_clean(camera_geom, image,
                                               **config['tailcut'])
                n_pixels = np.count_nonzero(signal_pixels)
                if n_pixels > 0:
                    num_islands, island_labels = number_of_islands(
                        camera_geom, signal_pixels)
                    n_pixels_on_island = np.bincount(
                        island_labels.astype(np.int))
                    n_pixels_on_island[
                        0] = 0  # first island is no-island and should not be considered
                    max_island_label = np.argmax(n_pixels_on_island)
                    signal_pixels[island_labels != max_island_label] = False

                    hillas = hillas_parameters(camera_geom[signal_pixels],
                                               image[signal_pixels])

                    dl1_container.fill_hillas(hillas)
                    dl1_container.set_timing_features(
                        camera_geom[signal_pixels], image[signal_pixels],
                        peak_time[signal_pixels], hillas)

                    dl1_container.set_leakage(camera_geom, image,
                                              signal_pixels)
                    dl1_container.set_concentration(camera_geom, image, hillas)
                    dl1_container.n_islands = num_islands
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    dl1_container.n_pixels = n_pixels
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(
                        np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width
                    dl1_container.length = length
                    dl1_container.r = np.sqrt(dl1_container.x**2 +
                                              dl1_container.y**2)

                else:
                    # for consistency with r0_to_dl1.py:
                    for key in dl1_container.keys():
                        dl1_container[key] = \
                            u.Quantity(0, dl1_container.fields[key].unit)

                    dl1_container.width = u.Quantity(np.nan, u.m)
                    dl1_container.length = u.Quantity(np.nan, u.m)
                    dl1_container.wl = u.Quantity(np.nan, u.m)

            for p in parameters_to_update:
                params[ii][p] = u.Quantity(dl1_container[p]).value

            output.root[dl1_params_lstcam_key][:] = params
    def _build_subarray_info(self, run):
        """
        constructs a SubarrayDescription object from the info in an
        MCRun

        Parameters
        ----------
        run: MCRun object

        Returns
        -------
        SubarrayDescription :
            instrumental information
        """
        subarray = SubarrayDescription("MonteCarloArray")
        runHeader = run.root.RunHeader

        tabFocalTel = runHeader.tabFocalTel.read()
        tabPosTelX = runHeader.tabPosTelX.read()
        tabPosTelY = runHeader.tabPosTelY.read()
        tabPosTelZ = runHeader.tabPosTelZ.read()

        tabPoslXYZ = np.ascontiguousarray(
            np.vstack((tabPosTelX, tabPosTelY, tabPosTelZ)).T)
        '''
        # Correspance HiPeData.Telscope.Type and camera name
        # 0  LSTCam, 1 NectarCam, 2 FlashCam, 3 SCTCam,
        # 4 ASTRICam, 5 DigiCam, 6 CHEC
        '''
        mapping_camera = {
            0: 'LSTCam',
            1: 'NectarCam',
            2: 'FlashCam',
            3: 'SCTCam',
            4: 'ASTRICam',
            5: 'DigiCam',
            6: 'CHEC'
        }

        mapping_telName = {
            0: 'LST',
            1: 'MST',
            2: 'MST',
            3: 'MST',
            4: 'SST-ASTRI',
            5: 'SST-1M',
            6: 'SST-2M'
        }

        for telNode in self.run.walk_nodes('/Tel', 'Group'):
            try:
                telType = uint64(telNode.telType.read())
                telIndex = uint64(telNode.telIndex.read())
                telId = uint64(telNode.telId.read())

                cameraName = mapping_camera[telType]
                telName = mapping_telName[telType]
                camera = CameraGeometry.from_name(cameraName)
                camera.cam_id = cameraName

                foclen = tabFocalTel[telIndex] * u.m

                tel_pos = tabPoslXYZ[telIndex] * u.m

                camera.pix_x = telNode.tabPixelX.read() * u.m
                camera.pix_y = telNode.tabPixelY.read() * u.m
                #camera.rotate(-90.0*u.deg)
                optic = OpticsDescription.from_name(telName)
                optic.equivalent_focal_length = foclen
                telescope_description = TelescopeDescription(telName,
                                                             telName,
                                                             optics=optic,
                                                             camera=camera)

                #tel.optics.mirror_area = mirror_area
                #tel.optics.num_mirror_tiles = num_tiles
                subarray.tels[telId] = telescope_description
                subarray.positions[telId] = tel_pos
            except tables.exceptions.NoSuchNodeError as e:
                pass

        return subarray
Exemplo n.º 24
0
    def __init__(self, **kwargs):
        """
        Constructor

        Parameters
        ----------
        kwargs: dict
            Parameters to be passed.
            NOTE: The file mask of the data to read can be passed with
            the 'input_url' parameter.
        """
        try:
            import uproot
        except ImportError:
            raise ImportError(
                "The 'uproot' package is required for the DLMAGICEventSource class."
            )

        self.file_list = glob.glob(kwargs['input_url'])
        self.file_list.sort()

        # Since EventSource can not handle file wild cards as input_url
        # We substitute the input_url with first file matching
        # the specified file mask.
        del kwargs['input_url']
        super().__init__(input_url=self.file_list[0], **kwargs)

        # get run number
        mask = r".*_za\d+to\d+_\d_(\d+)_([A-Z]+)_.*"
        parsed_info = re.findall(mask, self.file_list[0])
        self.run_number = parsed_info[0][0]

        # MAGIC telescope positions in m wrt. to the center of CTA simulations
        self.magic_tel_positions = {
            1: [-27.24, -146.66, 50.00] * u.m,
            2: [-96.44, -96.77, 51.00] * u.m
        }
        self.magic_tel_positions = self.magic_tel_positions
        # MAGIC telescope description
        optics = OpticsDescription.from_name('MAGIC')
        geom = CameraGeometry.from_name('MAGICCam')
        # Camera Readout for NectarCam used as a placeholder
        readout = CameraReadout(
            'MAGICCam',
            sampling_rate=u.Quantity(1, u.GHz),
            reference_pulse_shape=np.array([norm.pdf(np.arange(96), 48, 6)]),
            reference_pulse_sample_width=u.Quantity(1, u.ns))
        camera = CameraDescription('MAGICCam', geom, readout)
        self.magic_tel_description = TelescopeDescription(name='MAGIC',
                                                          tel_type='LST',
                                                          optics=optics,
                                                          camera=camera)
        self.magic_tel_descriptions = {
            1: self.magic_tel_description,
            2: self.magic_tel_description
        }
        self.magic_subarray = SubarrayDescription('MAGIC',
                                                  self.magic_tel_positions,
                                                  self.magic_tel_descriptions)
        # Open ROOT files
        self.calib_M1, self.calib_M2, self.star_M1, self.star_M2, self.superstar = None, None, None, None, None
        for file in self.file_list:
            uproot_file = uproot.open(file)
            if "_Y_" in file:
                if "_M1_" in file:
                    self.calib_M1 = uproot_file["Events"]
                    self.meta = uproot_file["RunHeaders"]
                elif "_M2_" in file:
                    self.calib_M2 = uproot_file["Events"]
            if "_I_" in file:
                if "_M1_" in file:
                    self.star_M1 = uproot_file["Events"]
                elif "_M2_" in file:
                    self.star_M2 = uproot_file["Events"]
            if "_S_" in file:
                self.superstar = uproot_file["Events"]
                self.meta = uproot_file["RunHeaders"]
        self._mc_header = self._parse_mc_header()
Exemplo n.º 25
0
def main():
    std_config = get_standard_config()

    log.setLevel(logging.INFO)
    handler = logging.StreamHandler()
    logging.getLogger().addHandler(handler)

    if args.config_file is not None:
        config = replace_config(std_config,
                                read_configuration_file(args.config_file))
    else:
        config = std_config

    if args.pedestal_cleaning:
        print("Pedestal cleaning")
        clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
        sigma = config[clean_method_name]['sigma']
        pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
        cleaning_params = get_cleaning_parameters(config, clean_method_name)
        pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
        log.info(
            f"Fraction of pixel cleaning thresholds above picture thr.:"
            f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}")
        picture_th = np.clip(pedestal_thresh, pic_th, None)
        log.info(f"Tailcut clean with pedestal threshold config used:"
                 f"{config['tailcuts_clean_with_pedestal_threshold']}")
    else:
        clean_method_name = 'tailcut'
        cleaning_params = get_cleaning_parameters(config, clean_method_name)
        picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
        log.info(f"Tailcut config used: {config['tailcut']}")

    use_only_main_island = True
    if "use_only_main_island" in config[clean_method_name]:
        use_only_main_island = config[clean_method_name][
            "use_only_main_island"]

    delta_time = None
    if "delta_time" in config[clean_method_name]:
        delta_time = config[clean_method_name]["delta_time"]

    foclen = OpticsDescription.from_name('LST').equivalent_focal_length
    cam_table = Table.read(args.input_file,
                           path="instrument/telescope/camera/LSTCam")
    camera_geom = CameraGeometry.from_table(cam_table)

    dl1_container = DL1ParametersContainer()
    parameters_to_update = [
        'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
        'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
        'concentration_pixel', 'leakage_intensity_width_1',
        'leakage_intensity_width_2', 'leakage_pixels_width_1',
        'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
        'n_pixels', 'wl', 'log_intensity'
    ]

    nodes_keys = get_dataset_keys(args.input_file)
    if args.noimage:
        nodes_keys.remove(dl1_images_lstcam_key)

    auto_merge_h5files([args.input_file],
                       args.output_file,
                       nodes_keys=nodes_keys)

    with tables.open_file(args.input_file, mode='r') as input:
        image_table = input.root[dl1_images_lstcam_key]
        dl1_params_input = input.root[dl1_params_lstcam_key].colnames
        disp_params = {
            'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
        }
        if set(dl1_params_input).intersection(disp_params):
            parameters_to_update.extend(disp_params)

        with tables.open_file(args.output_file, mode='a') as output:
            params = output.root[dl1_params_lstcam_key].read()
            for ii, row in enumerate(image_table):

                dl1_container.reset()

                image = row['image']
                peak_time = row['peak_time']

                signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
                                               boundary_th, isolated_pixels,
                                               min_n_neighbors)

                n_pixels = np.count_nonzero(signal_pixels)
                if n_pixels > 0:
                    num_islands, island_labels = number_of_islands(
                        camera_geom, signal_pixels)
                    n_pixels_on_island = np.bincount(
                        island_labels.astype(np.int64))
                    n_pixels_on_island[
                        0] = 0  # first island is no-island and should not be considered
                    max_island_label = np.argmax(n_pixels_on_island)
                    if use_only_main_island:
                        signal_pixels[
                            island_labels != max_island_label] = False

                    # if delta_time has been set, we require at least one
                    # neighbor within delta_time to accept a pixel in the image:
                    if delta_time is not None:
                        cleaned_pixel_times = peak_time
                        # makes sure only signal pixels are used in the time
                        # check:
                        cleaned_pixel_times[~signal_pixels] = np.nan
                        new_mask = apply_time_delta_cleaning(
                            camera_geom, signal_pixels, cleaned_pixel_times, 1,
                            delta_time)
                        signal_pixels = new_mask

                    # count the surviving pixels
                    n_pixels = np.count_nonzero(signal_pixels)

                    if n_pixels > 0:
                        hillas = hillas_parameters(camera_geom[signal_pixels],
                                                   image[signal_pixels])

                        dl1_container.fill_hillas(hillas)
                        dl1_container.set_timing_features(
                            camera_geom[signal_pixels], image[signal_pixels],
                            peak_time[signal_pixels], hillas)

                        dl1_container.set_leakage(camera_geom, image,
                                                  signal_pixels)
                        dl1_container.set_concentration(
                            camera_geom, image, hillas)
                        dl1_container.n_islands = num_islands
                        dl1_container.wl = dl1_container.width / dl1_container.length
                        dl1_container.n_pixels = n_pixels
                        width = np.rad2deg(
                            np.arctan2(dl1_container.width, foclen))
                        length = np.rad2deg(
                            np.arctan2(dl1_container.length, foclen))
                        dl1_container.width = width
                        dl1_container.length = length
                        dl1_container.log_intensity = np.log10(
                            dl1_container.intensity)

                if set(dl1_params_input).intersection(disp_params):
                    disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
                        dl1_container['x'].to_value(u.m),
                        dl1_container['y'].to_value(u.m), params['src_x'][ii],
                        params['src_y'][ii])

                    dl1_container['disp_dx'] = disp_dx
                    dl1_container['disp_dy'] = disp_dy
                    dl1_container['disp_norm'] = disp_norm
                    dl1_container['disp_angle'] = disp_angle
                    dl1_container['disp_sign'] = disp_sign

                for p in parameters_to_update:
                    params[ii][p] = u.Quantity(dl1_container[p]).value

            output.root[dl1_params_lstcam_key][:] = params
Exemplo n.º 26
0
def get_expected_source_pos(data, data_type, config):
    """Get expected source position for source-dependent analysis .

    Parameters:
    -----------
    data: Pandas DataFrame
    data_type: string ('mc_gamma','mc_proton','real_data')
    config: dictionnary containing configuration
    
    """

    #For gamma MC, expected source position is actual one for each event
    if data_type == 'mc_gamma':
        expected_src_pos_x_m = data['src_x'].values
        expected_src_pos_y_m = data['src_y'].values

    #For proton MC, nominal source position is one written in config file
    if data_type == 'mc_proton':
        focal_length = OpticsDescription.from_name(
            'LST').equivalent_focal_length
        expected_src_pos = utils.sky_to_camera(
            u.Quantity(data['mc_alt_tel'].values +
                       config['mc_nominal_source_x_deg'],
                       u.deg,
                       copy=False),
            u.Quantity(data['mc_az_tel'].values +
                       config['mc_nominal_source_y_deg'],
                       u.deg,
                       copy=False), focal_length,
            u.Quantity(data['mc_alt_tel'].values, u.deg, copy=False),
            u.Quantity(data['mc_az_tel'].values, u.deg, copy=False))

        expected_src_pos_x_m = expected_src_pos.x.to_value(u.m)
        expected_src_pos_y_m = expected_src_pos.y.to_value(u.m)

    # For real data
    if data_type == 'real_data':
        # source is always at the ceter of camera for ON mode
        if config.get('observation_mode') == 'on':
            expected_src_pos_x_m = np.zeros(len(data))
            expected_src_pos_y_m = np.zeros(len(data))

        # compute source position in camera coordinate event by event for wobble mode
        if config.get('observation_mode') == 'wobble':

            if 'source_name' in config:
                source_coord = SkyCoord.from_name(config.get('source_name'))
            else:
                source_coord = SkyCoord(config.get('source_ra'),
                                        config.get('source_dec'),
                                        frame="icrs",
                                        unit="deg")

            focal_length = OpticsDescription.from_name(
                'LST').equivalent_focal_length

            time = data['dragon_time']
            obstime = Time(time, scale='utc', format='unix')
            pointing_alt = u.Quantity(data['alt_tel'], u.rad, copy=False)
            pointing_az = u.Quantity(data['az_tel'], u.rad, copy=False)
            source_pos = utils.radec_to_camera(source_coord, obstime,
                                               pointing_alt, pointing_az,
                                               focal_length)

            expected_src_pos_x_m = source_pos.x.to_value(u.m)
            expected_src_pos_y_m = source_pos.y.to_value(u.m)

    return expected_src_pos_x_m, expected_src_pos_y_m
Exemplo n.º 27
0
def r0_to_dl1(input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
              output_filename=None,
              custom_config={}):
    """
    Chain r0 to dl1
    Save the extracted dl1 parameters in output_filename

    Parameters
    ----------
    input_filename: str
        path to input file, default: `gamma_test_large.simtel.gz`
    output_filename: str
        path to output file, default: `./` + basename(input_filename)
    config_file: path to a configuration file

    Returns
    -------

    """
    if output_filename is None:
        output_filename = ('dl1_' +
                           os.path.basename(input_filename).split('.')[0] +
                           '.h5')

    config = replace_config(standard_config, custom_config)

    custom_calibration = config["custom_calibration"]

    source = event_source(input_filename)
    source.allowed_tels = config["allowed_tels"]
    source.max_events = config["max_events"]

    cal = load_calibrator_from_config(config)

    dl1_container = DL1ParametersContainer()

    with HDF5TableWriter(filename=output_filename,
                         group_name='events',
                         overwrite=True) as writer:

        for i, event in enumerate(source):
            if i % 100 == 0:
                print(i)
            if not custom_calibration:
                cal(event)
                # for telescope_id, dl1 in event.dl1.tel.items():
            for ii, telescope_id in enumerate(event.r0.tels_with_data):
                if custom_calibration:
                    lst_calibration(event, telescope_id)

                try:
                    dl1_filled = get_dl1(event,
                                         telescope_id,
                                         dl1_container=dl1_container,
                                         custom_config=config)
                except HillasParameterizationError:
                    logging.exception(
                        'HillasParameterizationError in get_dl1()')
                    continue

                if dl1_filled is not None:

                    # Some custom def
                    dl1_container.wl = dl1_container.width / dl1_container.length
                    # Log10(Energy) in GeV
                    dl1_container.mc_energy = np.log10(event.mc.energy.value *
                                                       1e3)
                    dl1_container.intensity = np.log10(dl1_container.intensity)
                    dl1_container.gps_time = event.trig.gps_time.value

                    foclen = (event.inst.subarray.tel[telescope_id].optics.
                              equivalent_focal_length)
                    width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
                    length = np.rad2deg(
                        np.arctan2(dl1_container.length, foclen))
                    dl1_container.width = width.value
                    dl1_container.length = length.value

                    if width >= 0:
                        # Camera geometry
                        camera = event.inst.subarray.tel[telescope_id].camera
                        writer.write(camera.cam_id, [dl1_container])

    lst_focal = OpticsDescription.from_name('LST').equivalent_focal_length

    with pd.HDFStore(output_filename) as store:

        df = store['events/LSTCam']

        source_pos_in_camera = sky_to_camera(
            df.mc_alt.values * u.rad,
            df.mc_az.values * u.rad,
            lst_focal,
            df.mc_alt_tel.values * u.rad,
            df.mc_az_tel.values * u.rad,
        )
        disp_parameters = disp.disp(df.x.values * u.m, df.y.values * u.m,
                                    source_pos_in_camera.x,
                                    source_pos_in_camera.y)

        disp_df = pd.DataFrame(np.transpose(disp_parameters),
                               columns=[
                                   'disp_dx', 'disp_dy', 'disp_norm',
                                   'disp_angle', 'disp_sign'
                               ])
        disp_df['src_x'] = source_pos_in_camera.x.value
        disp_df['src_y'] = source_pos_in_camera.y.value

        store['events/LSTCam'] = pd.concat([store['events/LSTCam'], disp_df],
                                           axis=1)

    with HDF5TableWriter(filename=output_filename,
                         group_name="simulation",
                         mode="a") as writer:
        writer.write("run_config", [event.mcheader])