def __generator(self): data = DataContainer() data.meta['origin'] = 'hessio' data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for counter, array_event in enumerate(self.file_): # next lines are just for debugging self.array_event = array_event data.event_type = array_event['type'] # calibration events do not have an event id if data.event_type == 'calibration': event_id = -1 else: event_id = array_event['event_id'] data.inst.subarray = self._subarray_info obs_id = self.file_.header['run'] tels_with_data = set(array_event['telescope_events'].keys()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected trigger_information = array_event['trigger_information'] data.trig.tels_with_trigger = trigger_information['triggered_telescopes'] time_s, time_ns = trigger_information['gps_time'] data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') if data.event_type == 'data': self.fill_mc_information(data, array_event) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes telescope_events = array_event['telescope_events'] tracking_positions = array_event['tracking_positions'] for tel_id, telescope_event in telescope_events.items(): tel_index = self.file_.header['tel_id'].tolist().index(tel_id) telescope_description = self.file_.telescope_descriptions[tel_id] adc_samples = telescope_event.get('adc_samples') if adc_samples is None: adc_samples = telescope_event['adc_sums'][:, :, np.newaxis] r0 = data.r0.tel[tel_id] r0.waveform = adc_samples r0.num_samples = adc_samples.shape[-1] # We should not calculate stuff in an event source # if this is not needed, we calculate it for nothing r0.image = adc_samples.sum(axis=-1) pixel_lists = telescope_event['pixel_lists'] r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels'] if r0.num_trig_pix > 0: r0.trig_pix_id = pixel_lists[0]['pixel_list'] pixel_settings = telescope_description['pixel_settings'] n_pixel = r0.waveform.shape[-2] mc = data.mc.tel[tel_id] mc.dc_to_pe = array_event['laser_calibrations'][tel_id]['calib'] mc.pedestal = array_event['camera_monitorings'][tel_id]['pedestal'] mc.reference_pulse_shape = pixel_settings['ref_shape'].astype('float64') mc.meta['refstep'] = float(pixel_settings['ref_step']) mc.time_slice = float(pixel_settings['time_slice']) mc.photo_electron_image = ( array_event .get('photoelectrons', {}) .get(tel_index, {}) .get('photoelectrons', np.zeros(n_pixel, dtype='float32')) ) tracking_position = tracking_positions[tel_id] mc.azimuth_raw = tracking_position['azimuth_raw'] mc.altitude_raw = tracking_position['altitude_raw'] mc.azimuth_cor = tracking_position.get('azimuth_cor', 0) mc.altitude_cor = tracking_position.get('altitude_cor', 0) yield data
def _generator(self): with self.pyhessio.open_hessio(self.input_url) as file: # the container is initialized once, and data is replaced within # it after each yield counter = 0 eventstream = file.move_to_next_event() data = DataContainer() data.meta['origin'] = "hessio" # some hessio_event_source specific parameters data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for event_id in eventstream: if counter == 0: # subarray info is only available when an event is loaded, # so load it on the first event. data.inst.subarray = self._build_subarray_info(file) obs_id = file.get_run_number() tels_with_data = set(file.get_teldata_list()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected data.trig.tels_with_trigger = ( file.get_central_event_teltrg_list()) time_s, time_ns = file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = file.get_mc_shower_energy() * u.TeV data.mc.alt = Angle(file.get_mc_shower_altitude(), u.rad) data.mc.az = Angle(file.get_mc_shower_azimuth(), u.rad) data.mc.core_x = file.get_mc_event_xcore() * u.m data.mc.core_y = file.get_mc_event_ycore() * u.m first_int = file.get_mc_shower_h_first_int() * u.m data.mc.h_first_int = first_int data.mc.x_max = file.get_mc_shower_xmax() * u.g / (u.cm**2) data.mc.shower_primary_id = file.get_mc_shower_primary_id() # mc run header data data.mcheader.run_array_direction = Angle( file.get_mc_run_array_direction() * u.rad) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes for tel_id in tels_with_data: # event.mc.tel[tel_id] = MCCameraContainer() data.mc.tel[tel_id].dc_to_pe = file.get_calibration(tel_id) data.mc.tel[tel_id].pedestal = file.get_pedestal(tel_id) data.r0.tel[tel_id].waveform = ( file.get_adc_sample(tel_id)) if data.r0.tel[tel_id].waveform.size == 0: # To handle ASTRI and dst files data.r0.tel[tel_id].waveform = ( file.get_adc_sum(tel_id)[..., None]) data.r0.tel[tel_id].image = file.get_adc_sum(tel_id) data.r0.tel[ tel_id].num_trig_pix = file.get_num_trig_pixels(tel_id) data.r0.tel[tel_id].trig_pix_id = file.get_trig_pixels( tel_id) data.mc.tel[tel_id].reference_pulse_shape = ( file.get_ref_shapes(tel_id)) nsamples = file.get_event_num_samples(tel_id) if nsamples <= 0: nsamples = 1 data.r0.tel[tel_id].num_samples = nsamples # load the data per telescope/pixel hessio_mc_npe = file.get_mc_number_photon_electron(tel_id) data.mc.tel[tel_id].photo_electron_image = hessio_mc_npe data.mc.tel[tel_id].meta['refstep'] = ( file.get_ref_step(tel_id)) data.mc.tel[tel_id].time_slice = ( file.get_time_slice(tel_id)) data.mc.tel[tel_id].azimuth_raw = ( file.get_azimuth_raw(tel_id)) data.mc.tel[tel_id].altitude_raw = ( file.get_altitude_raw(tel_id)) data.mc.tel[tel_id].azimuth_cor = ( file.get_azimuth_cor(tel_id)) data.mc.tel[tel_id].altitude_cor = ( file.get_altitude_cor(tel_id)) yield data counter += 1 return
def __generator(self): data = DataContainer() data.meta['origin'] = 'hessio' data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for counter, array_event in enumerate(self.file_): # next lines are just for debugging self.array_event = array_event event_id = array_event['event_id'] data.inst.subarray = self._subarray_info obs_id = self.file_.header['run'] tels_with_data = set(array_event['telescope_events'].keys()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected trigger_information = array_event['trigger_information'] mc_event = array_event['mc_event'] mc_shower = array_event['mc_shower'] data.trig.tels_with_trigger = trigger_information[ 'triggered_telescopes'] time_s, time_ns = trigger_information['gps_time'] data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = mc_shower['energy'] * u.TeV data.mc.alt = Angle(mc_shower['altitude'], u.rad) data.mc.az = Angle(mc_shower['azimuth'], u.rad) data.mc.core_x = mc_event['xcore'] * u.m data.mc.core_y = mc_event['ycore'] * u.m first_int = mc_shower['h_first_int'] * u.m data.mc.h_first_int = first_int data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2) data.mc.shower_primary_id = mc_shower['primary_id'] # mc run header data data.mcheader.run_array_direction = Angle( self.file_.header['direction'] * u.rad) mc_run_head = self.file_.mc_run_headers[-1] data.mcheader.corsika_version = mc_run_head['shower_prog_vers'] data.mcheader.simtel_version = mc_run_head['detector_prog_vers'] data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT data.mcheader.prod_site_B_declination = Angle( mc_run_head['B_declination'] * u.rad) data.mcheader.prod_site_B_inclination = Angle( mc_run_head['B_inclination'] * u.rad) data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m data.mcheader.spectral_index = mc_run_head['spectral_index'] # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes telescope_events = array_event['telescope_events'] tracking_positions = array_event['tracking_positions'] for tel_id, telescope_event in telescope_events.items(): telescope_description = self.file_.telescope_descriptions[ tel_id] data.mc.tel[tel_id].dc_to_pe = array_event[ 'laser_calibrations'][tel_id]['calib'] data.mc.tel[tel_id].pedestal = array_event[ 'camera_monitorings'][tel_id]['pedestal'] adc_samples = telescope_event.get('adc_samples') if adc_samples is None: adc_samples = telescope_event['adc_sums'][:, :, np.newaxis] data.r0.tel[tel_id].waveform = adc_samples data.r0.tel[tel_id].num_samples = adc_samples.shape[-1] # We should not calculate stuff in an event source # if this is not needed, we calculate it for nothing data.r0.tel[tel_id].image = adc_samples.sum(axis=-1) pixel_lists = telescope_event['pixel_lists'] data.r0.tel[tel_id].num_trig_pix = pixel_lists.get( 0, {'pixels': 0})['pixels'] if data.r0.tel[tel_id].num_trig_pix > 0: data.r0.tel[tel_id].trig_pix_id = pixel_lists[0][ 'pixel_list'] pixel_settings = telescope_description['pixel_settings'] data.mc.tel[tel_id].reference_pulse_shape = pixel_settings[ 'refshape'].astype('float64') data.mc.tel[tel_id].meta['refstep'] = float( pixel_settings['ref_step']) data.mc.tel[tel_id].time_slice = float( pixel_settings['time_slice']) n_pixel = data.r0.tel[tel_id].waveform.shape[-2] data.mc.tel[tel_id].photo_electron_image = array_event.get( 'photoelectrons', {}).get(tel_id) if data.mc.tel[tel_id].photo_electron_image is None: data.mc.tel[tel_id].photo_electron_image = np.zeros( (n_pixel, ), dtype='i2') tracking_position = tracking_positions[tel_id] data.mc.tel[tel_id].azimuth_raw = tracking_position[ 'azimuth_raw'] data.mc.tel[tel_id].altitude_raw = tracking_position[ 'altitude_raw'] data.mc.tel[tel_id].azimuth_cor = tracking_position.get( 'azimuth_cor', 0) data.mc.tel[tel_id].altitude_cor = tracking_position.get( 'altitude_cor', 0) yield data
def __generator(self): data = DataContainer() data.meta['origin'] = 'hessio' data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for counter, array_event in enumerate(self.file_): # next lines are just for debugging self.array_event = array_event data.event_type = array_event['type'] # calibration events do not have an event id if data.event_type == 'calibration': event_id = -1 else: event_id = array_event['event_id'] data.inst.subarray = self._subarray_info obs_id = self.file_.header['run'] tels_with_data = set(array_event['telescope_events'].keys()) data.count = counter data.index.obs_id = obs_id data.index.event_id = event_id data.r0.obs_id = obs_id # deprecated data.r0.event_id = event_id # deprecated data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id # deprecated data.r1.event_id = event_id # deprecated data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id # deprecated data.dl0.event_id = event_id # deprecated data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected trigger_information = array_event['trigger_information'] data.trig.tels_with_trigger = trigger_information[ 'triggered_telescopes'] time_s, time_ns = trigger_information['gps_time'] data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') if data.event_type == 'data': self.fill_mc_information(data, array_event) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes telescope_events = array_event['telescope_events'] tracking_positions = array_event['tracking_positions'] for tel_id, telescope_event in telescope_events.items(): tel_index = self.file_.header['tel_id'].tolist().index(tel_id) telescope_description = self.file_.telescope_descriptions[ tel_id] adc_samples = telescope_event.get('adc_samples') if adc_samples is None: adc_samples = telescope_event['adc_sums'][:, :, np.newaxis] _, n_pixels, n_samples = adc_samples.shape pixel_settings = telescope_description['pixel_settings'] mc = data.mc.tel[tel_id] mc.dc_to_pe = array_event['laser_calibrations'][tel_id][ 'calib'] mc.pedestal = array_event['camera_monitorings'][tel_id][ 'pedestal'] mc.reference_pulse_shape = pixel_settings['ref_shape'].astype( 'float64') mc.meta['refstep'] = float(pixel_settings['ref_step']) mc.time_slice = float(pixel_settings['time_slice']) mc.photo_electron_image = (array_event.get( 'photoelectrons', {}).get(tel_index, {}).get('photoelectrons', np.zeros(n_pixels, dtype='float32'))) tracking_position = tracking_positions[tel_id] mc.azimuth_raw = tracking_position['azimuth_raw'] mc.altitude_raw = tracking_position['altitude_raw'] mc.azimuth_cor = tracking_position.get('azimuth_cor', np.nan) mc.altitude_cor = tracking_position.get('altitude_cor', np.nan) if np.isnan(mc.azimuth_cor): data.pointing[tel_id].azimuth = u.Quantity( mc.azimuth_raw, u.rad) else: data.pointing[tel_id].azimuth = u.Quantity( mc.azimuth_cor, u.rad) if np.isnan(mc.altitude_cor): data.pointing[tel_id].altitude = u.Quantity( mc.altitude_raw, u.rad) else: data.pointing[tel_id].altitude = u.Quantity( mc.altitude_cor, u.rad) r0 = data.r0.tel[tel_id] r1 = data.r1.tel[tel_id] r0.waveform = adc_samples r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration( adc_samples, mc.pedestal, mc.dc_to_pe, self.gain_selector) pixel_lists = telescope_event['pixel_lists'] r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels'] if r0.num_trig_pix > 0: r0.trig_pix_id = pixel_lists[0]['pixel_list'] yield data
def iter_pedestal_events(self, telescope): """ Pedestal event generator. Yields DataContainer instances, filled with the read event data. Parameters ---------- telescope: str The telescope for which to return events. Can be either "M1" or "M2". Returns ------- """ counter = 0 telescope = telescope.upper() # Data container - is initialized once, and data is replaced within it after each yield data = DataContainer() data.meta['origin'] = "MAGIC" data.meta['input_url'] = self.input_url data.meta['is_simulation'] = False # Telescopes with data: tels_in_file = ["M1", "M2"] if telescope not in tels_in_file: raise ValueError("Specified telescope {:s} is not in the allowed list {}".format(telescope, tels_in_file)) tel_i = tels_in_file.index(telescope) tels_with_data = {tel_i + 1, } current_run = None # Loop over the available data runs for run_number in self.run_numbers: # Removing the previously read data run from memory if current_run is not None: if 'data' in current_run: del current_run['data'] # Setting the new active run current_run = self._set_active_run(run_number) if telescope == 'M1': n_events = current_run['data'].n_pedestal_events_m1 else: n_events = current_run['data'].n_pedestal_events_m2 # Loop over the events for event_i in range(n_events): # Event and run ids event_order_number = current_run['data'].pedestal_ids[telescope][event_i] event_id = current_run['data'].event_data[telescope]['stereo_event_number'][event_order_number] obs_id = current_run['number'] # Reading event data event_data = current_run['data'].get_pedestal_event_data(event_i, telescope=telescope) # Event counter data.count = counter # Setting up the R0 container data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tel.clear() # Setting up the R1 container data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tel.clear() # Setting up the DL0 container data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tel.clear() # Creating the telescope pointing container pointing = TelescopePointingContainer() pointing.azimuth = np.deg2rad(event_data['pointing_az']) * u.rad pointing.altitude = np.deg2rad(90 - event_data['pointing_zd']) * u.rad pointing.ra = np.deg2rad(event_data['pointing_ra']) * u.rad pointing.dec = np.deg2rad(event_data['pointing_dec']) * u.rad # Adding the pointing container to the event data data.pointing[tel_i + 1] = pointing # Adding event charge and peak positions per pixel data.dl1.tel[tel_i + 1].image = event_data['image'] data.dl1.tel[tel_i + 1].peakpos = event_data['peak_pos'] # data.dl1.tel[tel_i + 1].badpixels = np.array( # file['dl1/tel' + str(i_tel + 1) + '/badpixels'], dtype=np.bool) # Adding the event arrival time time_tmp = Time(event_data['mjd'], scale='utc', format='mjd') data.trig.gps_time = Time(time_tmp, format='unix', scale='utc', precision=9) # Setting the telescopes with data data.r0.tels_with_data = tels_with_data data.r1.tels_with_data = tels_with_data data.dl0.tels_with_data = tels_with_data data.trig.tels_with_trigger = tels_with_data # Setting the instrument sub-array data.inst.subarray = self.magic_subarray yield data counter += 1 return
def _generator(self): with self.pyhessio.open_hessio(self.input_url) as file: # the container is initialized once, and data is replaced within # it after each yield counter = 0 eventstream = file.move_to_next_event() data = DataContainer() data.meta['origin'] = "hessio" # some hessio_event_source specific parameters data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for event_id in eventstream: if counter == 0: # subarray info is only available when an event is loaded, # so load it on the first event. data.inst.subarray = self._build_subarray_info(file) obs_id = file.get_run_number() tels_with_data = set(file.get_teldata_list()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected data.trig.tels_with_trigger = ( file.get_central_event_teltrg_list()) time_s, time_ns = file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = file.get_mc_shower_energy() * u.TeV data.mc.alt = Angle(file.get_mc_shower_altitude(), u.rad) data.mc.az = Angle(file.get_mc_shower_azimuth(), u.rad) data.mc.core_x = file.get_mc_event_xcore() * u.m data.mc.core_y = file.get_mc_event_ycore() * u.m first_int = file.get_mc_shower_h_first_int() * u.m data.mc.h_first_int = first_int data.mc.x_max = file.get_mc_shower_xmax() * u.g / (u.cm**2) data.mc.shower_primary_id = file.get_mc_shower_primary_id() # mc run header data data.mcheader.run_array_direction = Angle( file.get_mc_run_array_direction() * u.rad) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes for tel_id in tels_with_data: mc = data.mc.tel[tel_id] r0 = data.r0.tel[tel_id] r1 = data.r1.tel[tel_id] pointing = data.pointing[tel_id] adc_samples = file.get_adc_sample(tel_id) if adc_samples.size == 0: adc_samples = file.get_adc_sum(tel_id)[..., None] dc_to_pe = file.get_calibration(tel_id) pedestal = file.get_pedestal(tel_id) r0.waveform = adc_samples r1.waveform, r1.selected_gain_channel = apply_simtel_r1_calibration( adc_samples, pedestal, dc_to_pe, self.gain_selector) mc.dc_to_pe = dc_to_pe mc.pedestal = pedestal r0.num_trig_pix = file.get_num_trig_pixels(tel_id) r0.trig_pix_id = file.get_trig_pixels(tel_id) mc.reference_pulse_shape = file.get_ref_shapes(tel_id) # load the data per telescope/pixel hessio_mc_npe = file.get_mc_number_photon_electron(tel_id) mc.photo_electron_image = hessio_mc_npe mc.meta['refstep'] = file.get_ref_step(tel_id) mc.time_slice = file.get_time_slice(tel_id) mc.azimuth_raw = file.get_azimuth_raw(tel_id) mc.altitude_raw = file.get_altitude_raw(tel_id) azimuth_cor = file.get_azimuth_cor(tel_id) altitude_cor = file.get_altitude_cor(tel_id) # hessioeventsource pass 0 if there is no altitude/azimuth correction if azimuth_cor == 0 and mc.azimuth_raw != 0: mc.azimuth_cor = np.nan pointing.azimuth = u.Quantity(mc.azimuth_raw, u.rad) else: mc.azimuth_cor = azimuth_cor pointing.azimuth = u.Quantity(azimuth_cor, u.rad) if altitude_cor == 0 and mc.altitude_raw != 0: mc.altitude_cor = np.nan pointing.altitude = u.Quantity(mc.altitude_raw, u.rad) else: mc.altitude_cor = altitude_cor pointing.altitude = u.Quantity(mc.altitude_cor, u.rad) yield data counter += 1 return
def iter_stereo_events(self): """ Stereo event generator. Yields DataContainer instances, filled with the read event data. Returns ------- """ counter = 0 # Data container - is initialized once, and data is replaced within it after each yield data = DataContainer() data.meta['origin'] = "MAGIC" data.meta['input_url'] = self.input_url data.meta['is_simulation'] = False # Telescopes with data: tels_in_file = ["m1", "m2"] tels_with_data = {1, 2} current_run = None # Loop over the available data runs for run_number in self.run_numbers: # Removing the previously read data run from memory if current_run is not None: if 'data' in current_run: del current_run['data'] # Setting the new active run current_run = self._set_active_run(run_number) # Loop over the events for event_i in range(current_run['data'].n_stereo_events): # Event and run ids event_order_number = current_run['data'].stereo_ids[event_i][0] event_id = current_run['data'].event_data['M1']['stereo_event_number'][event_order_number] obs_id = current_run['number'] # Reading event data event_data = current_run['data'].get_stereo_event_data(event_i) # Event counter data.count = counter # Setting up the R0 container data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tel.clear() # Setting up the R1 container data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tel.clear() # Setting up the DL0 container data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tel.clear() # Filling the DL1 container with the event data for tel_i, tel_id in enumerate(tels_in_file): # Creating the telescope pointing container pointing = TelescopePointingContainer() pointing.azimuth = np.deg2rad(event_data['pointing_az']) * u.rad pointing.altitude = np.deg2rad(90 - event_data['pointing_zd']) * u.rad pointing.ra = np.deg2rad(event_data['pointing_ra']) * u.rad pointing.dec = np.deg2rad(event_data['pointing_dec']) * u.rad # Adding the pointing container to the event data data.pointing[tel_i + 1] = pointing # Adding event charge and peak positions per pixel data.dl1.tel[tel_i + 1].image = event_data['{:s}_image'.format(tel_id)] data.dl1.tel[tel_i + 1].peakpos = event_data['{:s}_peak_pos'.format(tel_id)] # data.dl1.tel[i_tel + 1].badpixels = np.array( # file['dl1/tel' + str(i_tel + 1) + '/badpixels'], dtype=np.bool) # Adding the event arrival time time_tmp = Time(event_data['mjd'], scale='utc', format='mjd') data.trig.gps_time = Time(time_tmp, format='unix', scale='utc', precision=9) # Setting the telescopes with data data.r0.tels_with_data = tels_with_data data.r1.tels_with_data = tels_with_data data.dl0.tels_with_data = tels_with_data data.trig.tels_with_trigger = tels_with_data # Setting the instrument sub-array data.inst.subarray = self.magic_subarray yield data counter += 1 return
def _generator(self): with self.pyhessio.open_hessio(self.input_url) as file: # the container is initialized once, and data is replaced within # it after each yield counter = 0 eventstream = file.move_to_next_event() data = DataContainer() data.meta['origin'] = "hessio" # some hessio_event_source specific parameters data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for event_id in eventstream: if counter == 0: # subarray info is only available when an event is loaded, # so load it on the first event. data.inst.subarray = self._build_subarray_info(file) obs_id = file.get_run_number() tels_with_data = set(file.get_teldata_list()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected data.trig.tels_with_trigger = (file. get_central_event_teltrg_list()) time_s, time_ns = file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = file.get_mc_shower_energy() * u.TeV data.mc.alt = Angle(file.get_mc_shower_altitude(), u.rad) data.mc.az = Angle(file.get_mc_shower_azimuth(), u.rad) data.mc.core_x = file.get_mc_event_xcore() * u.m data.mc.core_y = file.get_mc_event_ycore() * u.m first_int = file.get_mc_shower_h_first_int() * u.m data.mc.h_first_int = first_int data.mc.x_max = file.get_mc_shower_xmax() * u.g / (u.cm**2) data.mc.shower_primary_id = file.get_mc_shower_primary_id() # mc run header data data.mcheader.run_array_direction = Angle( file.get_mc_run_array_direction() * u.rad ) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes for tel_id in tels_with_data: # event.mc.tel[tel_id] = MCCameraContainer() data.mc.tel[tel_id].dc_to_pe = file.get_calibration(tel_id) data.mc.tel[tel_id].pedestal = file.get_pedestal(tel_id) data.r0.tel[tel_id].waveform = (file. get_adc_sample(tel_id)) if data.r0.tel[tel_id].waveform.size == 0: # To handle ASTRI and dst files data.r0.tel[tel_id].waveform = (file. get_adc_sum(tel_id)[..., None]) data.r0.tel[tel_id].image = file.get_adc_sum(tel_id) data.r0.tel[tel_id].num_trig_pix = file.get_num_trig_pixels(tel_id) data.r0.tel[tel_id].trig_pix_id = file.get_trig_pixels(tel_id) data.mc.tel[tel_id].reference_pulse_shape = (file. get_ref_shapes(tel_id)) nsamples = file.get_event_num_samples(tel_id) if nsamples <= 0: nsamples = 1 data.r0.tel[tel_id].num_samples = nsamples # load the data per telescope/pixel hessio_mc_npe = file.get_mc_number_photon_electron(tel_id) data.mc.tel[tel_id].photo_electron_image = hessio_mc_npe data.mc.tel[tel_id].meta['refstep'] = (file. get_ref_step(tel_id)) data.mc.tel[tel_id].time_slice = (file. get_time_slice(tel_id)) data.mc.tel[tel_id].azimuth_raw = (file. get_azimuth_raw(tel_id)) data.mc.tel[tel_id].altitude_raw = (file. get_altitude_raw(tel_id)) data.mc.tel[tel_id].azimuth_cor = (file. get_azimuth_cor(tel_id)) data.mc.tel[tel_id].altitude_cor = (file. get_altitude_cor(tel_id)) yield data counter += 1 return
def _generator(self): # HiPeData arranges data per telescope and not by event like simtel # We need to first create a dictionary. # key -> EventId, value -> List of event from triggered telescopes self.events = _convert_per_events_to_per_telescope(self.run) # the container is initialized once, and data is replaced within # it after each yield counter = 0 data = DataContainer() data.meta['origin'] = "hipehdf5" # some hessio_event_source specific parameters data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events ''' MC data are valid for the whole run ''' data.mc.tel.clear() # clear the previous telescopes for telNode in self.run.walk_nodes('/Tel', 'Group'): try: tel_id = uint64(telNode.telId.read()) data.mc.tel[tel_id].dc_to_pe = telNode.tabGain.read() data.mc.tel[tel_id].pedestal = telNode.tabPed.read() data.mc.tel[ tel_id].reference_pulse_shape = telNode.tabRefShape.read() except tables.exceptions.NoSuchNodeError as e: pass corsika = self.run.root.Corsika tabEvent = corsika.tabCorsikaEvent.read() tabEventId = tabEvent["eventId"] runHeader = self.run.root.RunHeader azimuth = runHeader.azimuth.read() for event_id, event_list in self.events.items(): if counter == 0: # subarray info is only available when an event is loaded, # so load it on the first event. data.inst.subarray = self._build_subarray_info(self.run) obs_id = 0 tels_with_data = set([info[0] for info in event_list]) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected #data.trig.tels_with_trigger = set(tels_with_data) data.trig.tels_with_trigger = array(list(tels_with_data), dtype=int16) # indexSimu = np.where(tabEventId == 2498406) ''' time_s, time_ns = file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') ''' data.mc.energy = tabEvent["energy"] * u.TeV data.mc.alt = Angle(tabEvent["alt"], u.rad) data.mc.az = Angle(tabEvent["az"], u.rad) data.mc.core_x = tabEvent["coreX"] * u.m data.mc.core_y = tabEvent["coreY"] * u.m data.mc.h_first_int = tabEvent["h_first_int"] * u.m data.mc.x_max = tabEvent["xmax"] * u.g / (u.cm**2) data.mc.shower_primary_id = tabEvent["showerPrimaryId"] data.mcheader.run_array_direction = Angle(azimuth * u.rad) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() for telescopeId, telescopeIndex, event in event_list: telNode = self.run.get_node("/Tel", 'Tel_' + str(telescopeIndex)) matWaveform = telNode.waveform.read(event, event + 1) matWaveform = matWaveform["waveform"] matSignalPS = matWaveform[0].swapaxes(1, 2) data.r0.tel[telescopeId].waveform = matSignalPS _, _, n_samples = matSignalPS.shape ped = data.mc.tel[telescopeId].pedestal[..., np.newaxis] / n_samples gain = data.mc.tel[telescopeId].dc_to_pe[..., np.newaxis] data.r1.tel[telescopeId].waveform = (matSignalPS - ped) * gain #data.r0.tel[telescopeId].image= matSignalPS.sum(axis=2) #data.r0.tel[telescopeId].num_trig_pix = file.get_num_trig_pixels(telescopeId) #data.r0.tel[telescopeId].trig_pix_id = file.get_trig_pixels(telescopeId) yield data counter += 1 return
def __generator(self): data = DataContainer() data.meta['origin'] = 'hessio' data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events for counter, array_event in enumerate(self.file_): # next lines are just for debugging self.array_event = array_event event_id = array_event['event_id'] data.inst.subarray = self._subarray_info obs_id = self.file_.header['run'] tels_with_data = set(array_event['telescope_events'].keys()) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected trigger_information = array_event['trigger_information'] mc_event = array_event['mc_event'] mc_shower = array_event['mc_shower'] data.trig.tels_with_trigger = trigger_information['triggered_telescopes'] time_s, time_ns = trigger_information['gps_time'] data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') data.mc.energy = mc_shower['energy'] * u.TeV data.mc.alt = Angle(mc_shower['altitude'], u.rad) data.mc.az = Angle(mc_shower['azimuth'], u.rad) data.mc.core_x = mc_event['xcore'] * u.m data.mc.core_y = mc_event['ycore'] * u.m first_int = mc_shower['h_first_int'] * u.m data.mc.h_first_int = first_int data.mc.x_max = mc_shower['xmax'] * u.g / (u.cm**2) data.mc.shower_primary_id = mc_shower['primary_id'] # mc run header data data.mcheader.run_array_direction = Angle( self.file_.header['direction'] * u.rad ) mc_run_head = self.file_.mc_run_headers[-1] data.mcheader.corsika_version = mc_run_head['shower_prog_vers'] data.mcheader.simtel_version = mc_run_head['detector_prog_vers'] data.mcheader.energy_range_min = mc_run_head['E_range'][0] * u.TeV data.mcheader.energy_range_max = mc_run_head['E_range'][1] * u.TeV data.mcheader.prod_site_B_total = mc_run_head['B_total'] * u.uT data.mcheader.prod_site_B_declination = Angle( mc_run_head['B_declination'] * u.rad) data.mcheader.prod_site_B_inclination = Angle( mc_run_head['B_inclination'] * u.rad) data.mcheader.prod_site_alt = mc_run_head['obsheight'] * u.m data.mcheader.spectral_index = mc_run_head['spectral_index'] # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() data.mc.tel.clear() # clear the previous telescopes telescope_events = array_event['telescope_events'] tracking_positions = array_event['tracking_positions'] for tel_id, telescope_event in telescope_events.items(): telescope_description = self.file_.telescope_descriptions[tel_id] data.mc.tel[tel_id].dc_to_pe = array_event['laser_calibrations'][tel_id]['calib'] data.mc.tel[tel_id].pedestal = array_event['camera_monitorings'][tel_id]['pedestal'] adc_samples = telescope_event.get('adc_samples') if adc_samples is None: adc_samples = telescope_event['adc_sums'][:, :, np.newaxis] data.r0.tel[tel_id].waveform = adc_samples data.r0.tel[tel_id].num_samples = adc_samples.shape[-1] # We should not calculate stuff in an event source # if this is not needed, we calculate it for nothing data.r0.tel[tel_id].image = adc_samples.sum(axis=-1) pixel_lists = telescope_event['pixel_lists'] data.r0.tel[tel_id].num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels'] if data.r0.tel[tel_id].num_trig_pix > 0: data.r0.tel[tel_id].trig_pix_id = pixel_lists[0]['pixel_list'] pixel_settings = telescope_description['pixel_settings'] data.mc.tel[tel_id].reference_pulse_shape = pixel_settings['refshape'].astype('float64') data.mc.tel[tel_id].meta['refstep'] = float(pixel_settings['ref_step']) data.mc.tel[tel_id].time_slice = float(pixel_settings['time_slice']) n_pixel = data.r0.tel[tel_id].waveform.shape[-2] data.mc.tel[tel_id].photo_electron_image = array_event.get( 'photoelectrons', {} ).get(tel_id) if data.mc.tel[tel_id].photo_electron_image is None: data.mc.tel[tel_id].photo_electron_image = np.zeros((n_pixel, ), dtype='i2') tracking_position = tracking_positions[tel_id] data.mc.tel[tel_id].azimuth_raw = tracking_position['azimuth_raw'] data.mc.tel[tel_id].altitude_raw = tracking_position['altitude_raw'] data.mc.tel[tel_id].azimuth_cor = tracking_position.get('azimuth_cor', 0) data.mc.tel[tel_id].altitude_cor = tracking_position.get('altitude_cor', 0) yield data
def _generator(self): # HiPeData arranges data per telescope and not by event like simtel # We need to first create a dictionary. # key -> EventId, value -> List of event from triggered telescopes self.events = _convert_per_events_to_per_telescope(self.run) # the container is initialized once, and data is replaced within # it after each yield counter = 0 data = DataContainer() data.meta['origin'] = "mchdf5v2PixelSlice" # some hessio_event_source specific parameters data.meta['input_url'] = self.input_url data.meta['max_events'] = self.max_events ''' MC data are valid for the whole run ''' data.mc.tel.clear() # clear the previous telescopes for telNode in self.run.walk_nodes('/r1', 'Group'): try: tel_id = uint64(telNode.telId.read()) data.mc.tel[tel_id].dc_to_pe = telNode.tabGain.read() pedestal = telNode.pedestal.read() pedestal = pedestal["pedestal"] data.mc.tel[tel_id].pedestal = pedestal[0] data.mc.tel[ tel_id].reference_pulse_shape = telNode.tabRefShape.read() except tables.exceptions.NoSuchNodeError as e: pass tabEvent = self.run.root.simulation.mc_event.read() tabEventId = tabEvent["event_id"] azimuth = self.run.root.simulation.run_config.col( "run_array_direction")[0] for event_id, event_list in self.events.items(): if counter == 0: # subarray info is only available when an event is loaded, # so load it on the first event. data.inst.subarray = self._build_subarray_info(self.run) obs_id = 0 tels_with_data = set([info[0] for info in event_list]) data.count = counter data.r0.obs_id = obs_id data.r0.event_id = event_id data.r0.tels_with_data = tels_with_data data.r1.obs_id = obs_id data.r1.event_id = event_id data.r1.tels_with_data = tels_with_data data.dl0.obs_id = obs_id data.dl0.event_id = event_id data.dl0.tels_with_data = tels_with_data # handle telescope filtering by taking the intersection of # tels_with_data and allowed_tels if len(self.allowed_tels) > 0: selected = tels_with_data & self.allowed_tels if len(selected) == 0: continue # skip event data.r0.tels_with_data = selected data.r1.tels_with_data = selected data.dl0.tels_with_data = selected #data.trig.tels_with_trigger = set(tels_with_data) data.trig.tels_with_trigger = array(list(tels_with_data), dtype=int16) #TODO : replace this by an astropy join indexSimu = np.where(tabEventId == event_id) ''' time_s, time_ns = file.get_central_event_gps_time() data.trig.gps_time = Time(time_s * u.s, time_ns * u.ns, format='unix', scale='utc') ''' data.mc.energy = tabEvent["mc_energy"][indexSimu] * u.TeV data.mc.alt = Angle(tabEvent["mc_alt"][indexSimu], u.rad) data.mc.az = Angle(tabEvent["mc_az"][indexSimu], u.rad) data.mc.core_x = tabEvent["mc_core_x"][indexSimu] * u.m data.mc.core_y = tabEvent["mc_core_y"][indexSimu] * u.m data.mc.h_first_int = tabEvent["mc_h_first_int"][indexSimu] * u.m data.mc.x_max = tabEvent["mc_x_max"][indexSimu] * u.g / (u.cm**2) data.mc.shower_primary_id = tabEvent["mc_shower_primary_id"][ indexSimu] data.mcheader.run_array_direction = Angle(azimuth * u.rad) # this should be done in a nicer way to not re-allocate the # data each time (right now it's just deleted and garbage # collected) data.r0.tel.clear() data.r1.tel.clear() data.dl0.tel.clear() data.dl1.tel.clear() for telescopeId, telescopeIndex, event in event_list: telNode = self.run.get_node("/r1", 'Tel_' + str(telescopeId)) matWaveform = telNode.waveformHi.read(event, event + 1) matWaveform = matWaveform["waveformHi"][0] matSignalPSHi = matWaveform try: waveformLo = telNode.waveformLo.read(event, event + 1) waveformLo = waveformLo["waveformLo"][0] matSignalPSLo = waveformLo tabHiLo = np.stack((matSignalPSHi, matSignalPSLo)) data.r0.tel[telescopeId].waveform = tabHiLo except Exception as e: print(e) data.r0.tel[telescopeId].waveform = np.expand_dims( matSignalPSHi, axis=0) #data.r0.tel[telescopeId].image= matSignalPSHi.sum(axis=2) #data.r0.tel[telescopeId].num_trig_pix = file.get_num_trig_pixels(telescopeId) #data.r0.tel[telescopeId].trig_pix_id = file.get_trig_pixels(telescopeId) yield data counter += 1 return