def test_grad_fit(): """ Simple test that gradient fitting gives expected answers for perfect gradient """ grad = 2 intercept = 1 timing = timing_parameters( pix_x=np.zeros(4) * u.deg, pix_y=np.arange(4) * u.deg, image=np.ones(4), peak_time=intercept * u.ns + grad * np.arange(4) * u.ns, rotation_angle=0 * u.deg ) # Test we get the values we put in back out again assert_allclose(timing.gradient, grad * u.ns / u.deg) assert_allclose(timing.intercept, intercept * u.deg) # Then try a different rotation angle rot_angle = 20 * u.deg timing_rot20 = timing_parameters( pix_x=np.zeros(4) * u.deg, pix_y=np.arange(4) * u.deg, image=np.ones(4), peak_time=intercept * u.ns + grad * np.arange(4) * u.ns, rotation_angle=rot_angle ) # Test the output again makes sense assert_allclose(timing_rot20.gradient, timing.gradient / np.cos(rot_angle)) assert_allclose(timing_rot20.intercept, timing.intercept)
def test_psi_0(): """ Simple test that gradient fitting gives expected answers for perfect gradient """ grad = 2.0 intercept = 1.0 deviation = 0.1 geom = CameraGeometry.from_name("LSTCam") hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg) random = np.random.RandomState(1) pulse_time = intercept + grad * geom.pix_x.value pulse_time += random.normal(0, deviation, geom.n_pixels) timing = timing_parameters(geom, image=np.ones(geom.n_pixels), pulse_time=pulse_time, hillas_parameters=hillas, cleaning_mask=np.ones(geom.n_pixels, dtype=bool)) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2) assert_allclose(timing.intercept, intercept, rtol=1e-2) assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_psi_20(): # Then try a different rotation angle grad = 2 intercept = 1 deviation = 0.1 geom = CameraGeometry.from_name("LSTCam") psi = 20 * u.deg hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=psi) random = np.random.RandomState(1) pulse_time = intercept + grad * (np.cos(psi) * geom.pix_x.value + np.sin(psi) * geom.pix_y.value) pulse_time += random.normal(0, deviation, geom.n_pixels) timing = timing_parameters(geom, image=np.ones(geom.n_pixels), pulse_time=pulse_time, hillas_parameters=hillas, cleaning_mask=np.ones(geom.n_pixels, dtype=bool)) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2) assert_allclose(timing.intercept, intercept, rtol=1e-2) assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_ignore_negative(): grad = 2.0 intercept = 1.0 deviation = 0.1 geom = CameraGeometry.from_name("LSTCam") hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg) random = np.random.RandomState(1) pulse_time = intercept + grad * geom.pix_x.value pulse_time += random.normal(0, deviation, geom.n_pixels) image = np.ones(geom.n_pixels) image[5:10] = -1.0 cleaning_mask = image >= 0 timing = timing_parameters( geom, image, pulse_time=pulse_time, hillas_parameters=hillas, cleaning_mask=cleaning_mask, ) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2) assert_allclose(timing.intercept, intercept, rtol=1e-2) assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_array_display(): """ check that we can do basic array display functionality """ from ctapipe.visualization.mpl_array import ArrayDisplay from ctapipe.image.timing_parameters import timing_parameters # build a test subarray: tels = dict() tel_pos = dict() for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m): tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam") tel_pos[ii + 1] = pos sub = SubarrayDescription(name="TestSubarray", tel_positions=tel_pos, tel_descriptions=tels) ad = ArrayDisplay(sub) ad.set_vector_rho_phi(1 * u.m, 90 * u.deg) # try setting a value vals = ones(sub.num_tels) ad.values = vals assert (vals == ad.values).all() # test using hillas params: hillas_dict = { 1: HillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg), 2: HillasParametersContainer(length=20000 * u.cm, psi="95deg"), } grad = 2 intercept = 1 geom = CameraGeometry.from_name("LSTCam") rot_angle = 20 * u.deg hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=rot_angle) timing_rot20 = timing_parameters( geom, image=ones(geom.n_pixels), pulse_time=intercept + grad * geom.pix_x.value, hillas_parameters=hillas, cleaning_mask=ones(geom.n_pixels, dtype=bool), ) gradient_dict = { 1: timing_rot20.slope.value, 2: timing_rot20.slope.value, } ad.set_vector_hillas( hillas_dict=hillas_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg, ) ad.set_line_hillas(hillas_dict, range=300) ad.add_labels() ad.remove_labels()
def set_timing_features(self, geom, image, pulse_time, hillas): try: # if np.polyfit fails (e.g. len(image) < deg + 1) timepars = time.timing_parameters(geom, image, pulse_time, hillas) self.time_gradient = timepars.slope.value self.intercept = timepars.intercept except ValueError: self.time_gradient = np.nan self.intercept = np.nan
def set_timing_features(self, geom, image, peakpos, hillas): try: peak_time = Quantity(peakpos) * u.Unit("ns") timepars = time.timing_parameters(geom, image, peak_time, hillas) self.time_gradient = timepars['slope'].value self.intercept = timepars['intercept'] except: pass
def test_array_display(): from ctapipe.visualization.mpl_array import ArrayDisplay from ctapipe.image.timing_parameters import timing_parameters # build a test subarray: tels = dict() tel_pos = dict() for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m): tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam") tel_pos[ii + 1] = pos sub = SubarrayDescription(name="TestSubarray", tel_positions=tel_pos, tel_descriptions=tels) ad = ArrayDisplay(sub) ad.set_vector_rho_phi(1 * u.m, 90 * u.deg) # try setting a value vals = ones(sub.num_tels) ad.values = vals assert (vals == ad.values).all() # test using hillas params: hillas_dict = { 1: HillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg), 2: HillasParametersContainer(length=20000 * u.cm, psi="95deg"), } grad = 2 intercept = 1 rot_angle = 20 * u.deg timing_rot20 = timing_parameters(pix_x=arange(4) * u.deg, pix_y=zeros(4) * u.deg, image=ones(4), peak_time=intercept * u.ns + grad * arange(4) * u.ns, rotation_angle=rot_angle) gradient_dict = { 1: timing_rot20.gradient.value, 2: timing_rot20.gradient.value, } ad.set_vector_hillas(hillas_dict=hillas_dict, length=500, time_gradient=gradient_dict, angle_offset=0 * u.deg) ad.set_line_hillas(hillas_dict, range=300) ad.add_labels() ad.remove_labels()
def test_ignore_negative(): grad = 2.0 intercept = 1.0 geom = CameraGeometry.from_name("LSTCam") hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg) image = np.ones(geom.n_pixels) image[5:10] = -1.0 timing = timing_parameters( geom, image, peakpos=intercept + grad * geom.pix_x.value, hillas_parameters=hillas, ) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit) assert_allclose(timing.intercept, intercept)
def test_psi_20(): # Then try a different rotation angle grad = 2 intercept = 1 geom = CameraGeometry.from_name("LSTCam") psi = 20 * u.deg hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=psi) timing = timing_parameters( geom, image=np.ones(geom.n_pixels), peakpos=intercept + grad * (np.cos(psi) * geom.pix_x.value + np.sin(psi) * geom.pix_y.value), hillas_parameters=hillas, ) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit) assert_allclose(timing.intercept, intercept)
def test_psi_0(): """ Simple test that gradient fitting gives expected answers for perfect gradient """ grad = 2.0 intercept = 1.0 geom = CameraGeometry.from_name("LSTCam") hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg) timing = timing_parameters( geom, image=np.ones(geom.n_pixels), peakpos=intercept + grad * geom.pix_x.value, hillas_parameters=hillas, ) # Test we get the values we put in back out again assert_allclose(timing.slope, grad / geom.pix_x.unit) assert_allclose(timing.intercept, intercept)
def calculate_image_features(telescope_id, event, dl1, config): ''' Performs cleaning and adds the following image parameters: - hillas - leakage - concentration - timing - number of islands Make sure to adapt cleaning levels to the used algorithm (-> config) - tailcuts: picture_thresh, picture_thresh, min_number_picture_neighbors - fact_image_cleaning: picture_threshold, boundary_threshold, min_number_neighbors, time_limit Returns: -------- TelescopeParameterContainer ''' array_event_id = event.dl0.event_id run_id = event.r0.obs_id telescope = event.inst.subarray.tels[telescope_id] image = dl1.image # might want to make the parameter names more consistent between methods if config.cleaning_method == 'tailcuts_clean': boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[ telescope.camera.cam_id ] mask = tailcuts_clean( telescope.camera, image, boundary_thresh=boundary_thresh, picture_thresh=picture_thresh, min_number_picture_neighbors=min_number_picture_neighbors, ) elif config.cleaning_method == 'fact_image_cleaning': boundary_threshold, picture_threshold, time_limit, min_number_neighbors = config.cleaning_level[ telescope.camera.cam_id ] mask = fact_image_cleaning( telescope.camera, image, dl1.pulse_time, boundary_threshhold=boundary_threshold, picture_threshold=picture_threshold, min_number_neighbors=min_number_neighbors, time_limit=time_limit, ) cleaned = image.copy() cleaned[~mask] = 0 logging.debug(f'calculating hillas for event {array_event_id, telescope_id}') hillas_container = hillas_parameters(telescope.camera, cleaned) hillas_container.prefix = '' logging.debug(f'calculating leakage for event {array_event_id, telescope_id}') leakage_container = leakage(telescope.camera, image, mask) leakage_container.prefix = '' logging.debug(f'calculating concentration for event {array_event_id, telescope_id}') concentration_container = concentration(telescope.camera, image, hillas_container) concentration_container.prefix = '' logging.debug(f'getting timing information for event {array_event_id, telescope_id}') timing_container = timing_parameters(telescope.camera, image, dl1.pulse_time, hillas_container) timing_container.prefix = '' # membership missing for now as it causes problems with the hdf5tablewriter # right now i dont need this anyway logging.debug(f'calculating num_islands for event {array_event_id, telescope_id}') num_islands, membership = number_of_islands(telescope.camera, mask) island_container = IslandContainer(num_islands=num_islands) island_container.prefix = '' num_pixel_in_shower = mask.sum() logging.debug(f'getting pointing container for event {array_event_id, telescope_id}') # ctapipe requires this to be rad pointing_container = TelescopePointingContainer( azimuth=event.mc.tel[telescope_id].azimuth_raw * u.rad, altitude=event.mc.tel[telescope_id].altitude_raw * u.rad, prefix='pointing', ) return TelescopeParameterContainer( telescope_id=telescope_id, run_id=run_id, array_event_id=array_event_id, leakage=leakage_container, hillas=hillas_container, concentration=concentration_container, pointing=pointing_container, timing=timing_container, islands=island_container, telescope_type_id=config.types_to_id[telescope.type], camera_type_id=config.names_to_id[telescope.camera.cam_id], focal_length=telescope.optics.equivalent_focal_length, mirror_area=telescope.optics.mirror_area, num_pixel_in_shower=num_pixel_in_shower, )
continue # Calculate image parameters hillas = hillas_parameters( geom, clean) # this one gives some warnings invalid value in sqrt foclen = event.inst.subarray.tel[ tel_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(hillas.width, foclen)) l = np.rad2deg(np.arctan2(hillas.length, foclen)) #Calculate Timing parameters peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns") timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean, peak_time, hillas.psi) if w >= 0: if fitsdata.size == 0: fitsdata = clean else: fitsdata = np.vstack([fitsdata, clean]) #Pixel content camtype.append(str(geom)) width = np.append(width, w.value) length = np.append(length, l.value) phi = np.append(phi, hillas.phi) psi = np.append(psi, hillas.psi) r = np.append(r, hillas.r) x = np.append(x, hillas.x) y = np.append(y, hillas.y)
def get_events(filename, storedata=False, concatenate=False, storeimg=False, outdir='./results/'): """ Read a Simtelarray file, extract pixels charge, calculate image parameters and timing parameters and store the result in an hdf5 file. Parameters: filename: str Name of the simtelarray file. storedata: boolean True: store extracted data in a hdf5 file concatenate: boolean True: store the extracted data at the end of an existing file storeimg: boolean True: store also pixel data outdir: srt Output directory """ #Particle type: particle_type = guess_type(filename) #Create data frame where DL2 data will be stored: features = [ 'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y', 'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi', 'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel', 'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX', 'SrcY', 'disp', 'hadroness' ] output = pd.DataFrame(columns=features) #Read LST1 events: source = EventSourceFactory.produce( input_url=filename, allowed_tels={1}) #Open Simtelarray file #Cleaning levels: level1 = {'LSTCam': 6.} level2 = level1.copy() # We use as second cleaning level just half of the first cleaning level for key in level2: level2[key] *= 0.5 log10pixelHGsignal = {} survived = {} imagedata = np.array([]) for key in level1: log10pixelHGsignal[key] = [] survived[key] = [] i = 0 for event in source: if i % 100 == 0: print("EVENT_ID: ", event.r0.event_id, "TELS: ", event.r0.tels_with_data, "MC Energy:", event.mc.energy) i = i + 1 ntels = len(event.r0.tels_with_data) ''' if i > 100: # for quick tests break ''' for ii, tel_id in enumerate(event.r0.tels_with_data): geom = event.inst.subarray.tel[tel_id].camera #Camera geometry tel_coords = event.inst.subarray.tel_coords[ event.inst.subarray.tel_indices[tel_id]] data = event.r0.tel[tel_id].waveform ped = event.mc.tel[tel_id].pedestal # the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray nsamples = data.shape[2] # total number of samples pedcorrectedsamples = data - np.atleast_3d( ped ) / nsamples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix integrator = LocalPeakIntegrator(None, None) integration, peakpos, window = integrator.extract_charge( pedcorrectedsamples ) # these are 2D matrices num_gains * num_pixels chan = 0 # high gain used for now... signals = integration[chan].astype(float) dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels signals *= dc2pe[chan] # Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary log10pixelHGsignal[str(geom)].extend( np.log10(signals) ) # This seems to be faster like this, with normal python lists # Apply image cleaning cleanmask = tailcuts_clean(geom, signals, picture_thresh=level1[str(geom)], boundary_thresh=level2[str(geom)], keep_isolated_pixels=False, min_number_picture_neighbors=1) survived[str(geom)].extend( cleanmask ) # This seems to be faster like this, with normal python lists clean = signals.copy() clean[ ~cleanmask] = 0.0 # set to 0 pixels which did not survive cleaning if np.max(clean) < 1.e-6: # skip images with no pixels continue # Calculate image parameters hillas = hillas_parameters( geom, clean) # this one gives some warnings invalid value in sqrt foclen = event.inst.subarray.tel[ tel_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(hillas.width, foclen)) l = np.rad2deg(np.arctan2(hillas.length, foclen)) #Calculate Timing parameters peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns") timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean, peak_time, hillas.psi) if w >= 0: if storeimg == True: if imagedata.size == 0: imagedata = clean else: imagedata = np.vstack([imagedata, clean]) #Pixel content width = w.value length = l.value phi = hillas.phi.value psi = hillas.psi.value r = hillas.r.value x = hillas.x.value y = hillas.y.value intensity = np.log10(hillas.intensity) skewness = hillas.skewness kurtosis = hillas.kurtosis #Store parameters from event and MC: ObsID = event.r0.obs_id EvID = event.r0.event_id mcEnergy = np.log10(event.mc.energy.value * 1e3) #Log10(Energy) in GeV mcAlt = event.mc.alt.value mcAz = event.mc.az.value mcCore_x = event.mc.core_x.value mcCore_y = event.mc.core_y.value mcHfirst = event.mc.h_first_int.value mcType = event.mc.shower_primary_id mcAztel = event.mcheader.run_array_direction[0].value mcAlttel = event.mcheader.run_array_direction[1].value mcXmax = event.mc.x_max.value GPStime = event.trig.gps_time.value impact = np.sqrt( (tel_coords.x.value - event.mc.core_x.value)**2 + (tel_coords.y.value - event.mc.core_y.value)**2) time_gradient = timepars[0].value intercept = timepars[1].value #Calculate Disp and Source position in camera coordinates tel = OpticsDescription.from_name( 'LST') #Telescope description focal_length = tel.equivalent_focal_length.value sourcepos = transformations.calc_CamSourcePos( mcAlt, mcAz, mcAlttel, mcAztel, focal_length) SrcX = sourcepos[0] SrcY = sourcepos[1] disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x, y) hadroness = 0 if particle_type == 'proton': hadroness = 1 eventdf = pd.DataFrame([[ ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y, mcHfirst, mcType, GPStime, width, length, width / length, phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel, mcAztel, impact, mcXmax, time_gradient, intercept, SrcX, SrcY, disp, hadroness ]], columns=features) output = output.append(eventdf, ignore_index=True) outfile = outdir + particle_type + '_events.hdf5' if storedata == True: if concatenate == False or (concatenate == True and np.DataSource().exists(outfile) == False): output.to_hdf(outfile, key=particle_type + "_events", mode="w") if storeimg == True: f = h5py.File(outfile, 'r+') f.create_dataset('images', data=imagedata) f.close() else: if storeimg == True: f = h5py.File(outfile, 'r') images = f['images'] del f['images'] images = np.vstack([images, imagedata]) f.close() saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") f = h5py.File(outfile, 'r+') f.create_dataset('images', data=images) f.close() else: saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") del source return output
def process_event(event, config): ''' Processes ''' reco_algorithm = config.reco_algorithm features = {} params = {} pointing_azimuth = {} pointing_altitude = {} tel_x = {} tel_y = {} tel_focal_lengths = {} cleaning_method = config.cleaning_method valid_cleaning_methods = ['tailcuts_clean', 'fact_image_cleaning'] if cleaning_method not in valid_cleaning_methods: print('Cleaning Method not implemented') print('Please use one of ', valid_cleaning_methods) return None for telescope_id, dl1 in event.dl1.tel.items(): camera = event.inst.subarray.tels[telescope_id].camera if camera.cam_id not in config.allowed_cameras: continue telescope_type_name = event.inst.subarray.tels[ telescope_id].optics.tel_type if cleaning_method == 'tailcuts_clean': boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[ camera.cam_id] mask = tailcuts_clean(camera, dl1.image[0], *config.cleaning_level[camera.cam_id]) elif cleaning_method == 'fact_image_cleaning': mask = fact_image_cleaning( camera, dl1.image[0], *config.cleaning_level_fact[camera.cam_id]) try: cleaned = dl1.image[0].copy() cleaned[~mask] = 0 hillas_container = hillas_parameters( camera, cleaned, ) params[telescope_id] = hillas_container except HillasParameterizationError: continue # probably wise to add try...except blocks here as well # Add more Features here (look what ctapipe can do, timing?) num_islands, island_labels = number_of_islands(camera, mask) island_dict = { 'num_islands': num_islands, 'island_labels': island_labels } leakage_container = leakage(camera, dl1.image[0], mask) timing_container = timing_parameters(camera, dl1.image[0], dl1.peakpos[0], hillas_container) pointing_azimuth[ telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad pointing_altitude[ telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0] tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1] telescope_description = event.inst.subarray.tel[telescope_id] tel_focal_lengths[ telescope_id] = telescope_description.optics.equivalent_focal_length d = { 'array_event_id': event.dl0.event_id, 'telescope_id': int(telescope_id), 'camera_name': camera.cam_id, 'camera_id': config.names_to_id[camera.cam_id], 'run_id': event.r0.obs_id, 'telescope_type_name': telescope_type_name, 'telescope_type_id': config.types_to_id[telescope_type_name], 'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw, 'pointing_altitude': event.mc.tel[telescope_id].altitude_raw, 'mirror_area': telescope_description.optics.mirror_area, 'focal_length': telescope_description.optics.equivalent_focal_length, } d.update(hillas_container.as_dict()) d.update(leakage_container.as_dict()) d.update(island_dict) d.update(timing_container.as_dict()) features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()}) if reco_algorithm == 'intersection': reco = HillasIntersection() array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1], az=event.mcheader.run_array_direction[0], frame='altaz') reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths, array_direction) elif reco_algorithm == 'planes': reco = HillasReconstructor() reconstruction = reco.predict(params, event.inst, pointing_altitude, pointing_azimuth) for telescope_id in event.dl1.tel.keys(): if telescope_id not in params: continue camera = event.inst.subarray.tels[telescope_id].camera if camera.cam_id not in config.allowed_cameras: continue pos = event.inst.subarray.positions[telescope_id] x, y = pos[0], pos[1] core_x = reconstruction.core_x core_y = reconstruction.core_y d = np.sqrt((core_x - x)**2 + (core_y - y)**2) features[telescope_id]['distance_to_core'] = d.value return pd.DataFrame(list(features.values())), reconstruction, params
boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors) # ignore images with less than 5 pixels after cleaning if clean.sum() < 5: continue # image parameters hillas_c = hillas_parameters(geom[clean], image[clean]) leakage_c = leakage(geom, image, clean) n_islands, island_ids = number_of_islands(geom, clean) timing_c = timing_parameters( geom[clean], image[clean], peakpos[clean], hillas_c, ) # store parameters for stereo reconstruction hillas_containers[telescope_id] = hillas_c # store timegradients for plotting # ASTRI has no timing in PROD3b, so we use skewness instead if geom.camera_name != 'ASTRICam': time_gradients[telescope_id] = timing_c.slope.value else: time_gradients[telescope_id] = hillas_c.skewness print(geom.camera_name, time_gradients[telescope_id]) # make sure each telescope get's an arrow
def set_timing_features(self, geom, image, pulse_time, hillas): peak_time = Quantity(pulse_time) * u.Unit("ns") timepars = time.timing_parameters(geom, image, peak_time, hillas) self.time_gradient = timepars.slope.value self.intercept = timepars.intercept
def get_events(filename, storedata=False, test=False, concatenate=False, storeimg=False, outdir='./results/'): """ Depreciated, use r0_to_dl1. Read a Simtelarray file, extract pixels charge, calculate image parameters and timing parameters and store the result in an hdf5 file. Parameters: ----------- filename: str Name of the simtelarray file. storedata: boolean True: store extracted data in a hdf5 file concatenate: boolean True: store the extracted data at the end of an existing file storeimg: boolean True: store also pixel data outdir: srt Output directory Returns: -------- pandas DataFrame: output """ from warnings import warn warn("Deprecated: use r0_to_dl1") #Particle type: particle_type = utils.guess_type(filename) #Create data frame where DL2 data will be stored: features = [ 'obs_id', 'event_id', 'mc_energy', 'mc_alt', 'mc_az', 'mc_core_x', 'mc_core_y', 'mc_h_first_int', 'mc_type', 'gps_time', 'width', 'length', 'wl', 'phi', 'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mc_alt_tel', 'mc_az_tel', 'mc_core_distance', 'mc_x_max', 'time_gradient', 'intercept', 'src_x', 'src_y', 'disp_norm', ] output = pd.DataFrame(columns=features) #Read LST1 events: source = event_source(input_url=filename, allowed_tels={1}) #Open Simtelarray file #Cleaning levels: level1 = {'LSTCam': 6.} level2 = level1.copy() # We use as second cleaning level just half of the first cleaning level for key in level2: level2[key] *= 0.5 log10pixelHGsignal = {} survived = {} imagedata = np.array([]) for key in level1: log10pixelHGsignal[key] = [] survived[key] = [] i = 0 for event in source: if i % 100 == 0: print("EVENT_ID: ", event.r0.event_id, "TELS: ", event.r0.tels_with_data, "MC Energy:", event.mc.energy) i = i + 1 ntels = len(event.r0.tels_with_data) if test == True and i > 1000: # for quick tests break for ii, tel_id in enumerate(event.r0.tels_with_data): geom = event.inst.subarray.tel[tel_id].camera #Camera geometry tel_coords = event.inst.subarray.tel_coords[ event.inst.subarray.tel_indices[tel_id]] data = event.r0.tel[tel_id].waveform ped = event.mc.tel[tel_id].pedestal # the pedestal is the #average (for pedestal events) of the *sum* of all samples, #from sim_telarray nsamples = data.shape[2] # total number of samples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples integrator = LocalPeakWindowSum() integration, pulse_time = integrator( pedcorrectedsamples ) # these are 2D matrices num_gains * num_pixels chan = 0 # high gain used for now... signals = integration[chan].astype(float) dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels signals *= dc2pe[chan] # Add all individual pixel signals to the numpy array of the # corresponding camera inside the log10pixelsignal dictionary log10pixelHGsignal[str(geom)].extend(np.log10(signals)) # Apply image cleaning cleanmask = tailcuts_clean(geom, signals, picture_thresh=level1[str(geom)], boundary_thresh=level2[str(geom)], keep_isolated_pixels=False, min_number_picture_neighbors=1) survived[str(geom)].extend(cleanmask) clean = signals.copy() clean[~cleanmask] = 0.0 # set to 0 pixels which did not # survive cleaning if np.max(clean) < 1.e-6: # skip images with no pixels continue # Calculate image parameters hillas = hillas_parameters(geom, clean) foclen = event.inst.subarray.tel[ tel_id].optics.equivalent_focal_length w = np.rad2deg(np.arctan2(hillas.width, foclen)) l = np.rad2deg(np.arctan2(hillas.length, foclen)) #Calculate Timing parameters peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns") timepars = time.timing_parameters(geom, clean, peak_time, hillas) if w >= 0: if storeimg == True: if imagedata.size == 0: imagedata = clean else: imagedata = np.vstack([imagedata, clean]) #Pixel content #Hillas parameters width = w.value length = l.value phi = hillas.phi.value psi = hillas.psi.value r = hillas.r.value x = hillas.x.value y = hillas.y.value intensity = np.log10(hillas.intensity) skewness = hillas.skewness kurtosis = hillas.kurtosis #MC data: obs_id = event.r0.obs_id event_id = event.r0.event_id mc_energy = np.log10(event.mc.energy.value * 1e3) #Log10(Energy) in GeV mc_alt = event.mc.alt.value mc_az = event.mc.az.value mc_core_x = event.mc.core_x.value mc_core_y = event.mc.core_y.value mc_h_first_int = event.mc.h_first_int.value mc_type = event.mc.shower_primary_id mc_az_tel = event.mcheader.run_array_direction[0].value mc_alt_tel = event.mcheader.run_array_direction[1].value mc_x_max = event.mc.x_max.value gps_time = event.trig.gps_time.value #Calculate mc_core_distance parameters mc_core_distance = np.sqrt( (tel_coords.x.value - event.mc.core_x.value)**2 + (tel_coords.y.value - event.mc.core_y.value)**2) #Timing parameters time_gradient = timepars['slope'].value intercept = timepars['intercept'] #Calculate disp_ and Source position in camera coordinates tel = OpticsDescription.from_name( 'LST') #Telescope description focal_length = tel.equivalent_focal_length.value sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel, mc_az_tel, focal_length) src_x = sourcepos[0] src_y = sourcepos[1] disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y) eventdf = pd.DataFrame([[ obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x, mc_core_y, mc_h_first_int, mc_type, gps_time, width, length, width / length, phi, psi, r, x, y, intensity, skewness, kurtosis, mc_alt_tel, mc_az_tel, mc_core_distance, mc_x_max, time_gradient, intercept, src_x, src_y, disp, mc_type ]], columns=features) output = output.append(eventdf, ignore_index=True) outfile = outdir + particle_type + '_events.hdf5' if storedata == True: if (concatenate == False or (concatenate == True and np.DataSource().exists(outfile) == False)): output.to_hdf(outfile, key=particle_type + "_events", mode="w") if storeimg == True: f = h5py.File(outfile, 'r+') f.create_dataset('images', data=imagedata) f.close() else: if storeimg == True: f = h5py.File(outfile, 'r') images = f['images'] del f['images'] images = np.vstack([images, imagedata]) f.close() saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") f = h5py.File(outfile, 'r+') f.create_dataset('images', data=images) f.close() else: saved = pd.read_hdf(outfile, key=particle_type + '_events') output = saved.append(output, ignore_index=True) output.to_hdf(outfile, key=particle_type + "_events", mode="w") del source return output
boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors ) # ignore images with less than 5 pixels after cleaning if clean.sum() < 5: continue # image parameters hillas_c = hillas_parameters(camera[clean], image[clean]) leakage_c = leakage(camera, image, clean) n_islands, island_ids = number_of_islands(camera, clean) timing_c = timing_parameters( camera[clean], image[clean], peakpos[clean], hillas_c ) # store parameters for stereo reconstruction hillas_containers[telescope_id] = hillas_c pointing_azimuth[telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad pointing_altitude[telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad # store timegradients for plotting # ASTRI has no timing in PROD3b, so we use skewness instead if camera.cam_id != 'ASTRICam': time_gradients[telescope_id] = timing_c.slope.value else: time_gradients[telescope_id] = hillas_c.skewness print(camera.cam_id, time_gradients[telescope_id])
def set_timing_features(self, geom, image, pulse_time, hillas): timepars = time.timing_parameters(geom, image, pulse_time, hillas) self.time_gradient = timepars.slope.value self.intercept = timepars.intercept