def get_dl1b_tailcut(dl1a_img, dl1a_pulse, config_path, use_main_island=True): cleaning_method = tailcuts_clean config = read_configuration_file(config_path) cleaning_parameters = config["tailcut"] dl1_container = DL1ParametersContainer() image = dl1a_img pulse_time = dl1a_pulse signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: # check the number of islands num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels) if use_main_island: n_pixels_on_island = np.bincount(island_labels.astype(np.int)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geometry[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features(camera_geometry[signal_pixels], image[signal_pixels], pulse_time[signal_pixels], hillas) set_converted_hillas_param(dl1_container, dl1_container.width, dl1_container.length) set_image_param(dl1_container, image, signal_pixels, hillas, n_pixels, num_islands) return dl1_container
def set_n_islands(self, geom, clean): n_islands, islands_mask = number_of_islands(geom, clean) self.n_islands = n_islands
def main(): std_config = get_standard_config() log.setLevel(logging.INFO) handler = logging.StreamHandler() logging.getLogger().addHandler(handler) if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config log.info(f"Tailcut config used: {config['tailcut']}") foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = list(HillasParametersContainer().keys()) parameters_to_update.extend([ 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'log_intensity' ]) nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] dl1_params_input = input.root[dl1_params_lstcam_key].colnames disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'} if set(dl1_params_input).intersection(disp_params): parameters_to_update.extend(disp_params) with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): dl1_container.reset() image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut']) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands(camera_geom, signal_pixels) n_pixels_on_island = np.bincount(island_labels.astype(np.int)) n_pixels_on_island[0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features(camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration(camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.log_intensity = np.log10(dl1_container.intensity) if set(dl1_params_input).intersection(disp_params): disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp( dl1_container['x'].to_value(u.m), dl1_container['y'].to_value(u.m), params['src_x'][ii], params['src_y'][ii] ) dl1_container['disp_dx'] = disp_dx dl1_container['disp_dy'] = disp_dy dl1_container['disp_norm'] = disp_norm dl1_container['disp_angle'] = disp_angle dl1_container['disp_sign'] = disp_sign for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def main(): std_config = get_standard_config() if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config print(config['tailcut']) foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = list(HillasParametersContainer().keys()) parameters_to_update.extend([ 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'r', ]) nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): if ii % 10000 == 0: print(ii) image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut']) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands( camera_geom, signal_pixels) n_pixels_on_island = np.bincount( island_labels.astype(np.int)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features( camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration(camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2) else: # for consistency with r0_to_dl1.py: for key in dl1_container.keys(): dl1_container[key] = \ u.Quantity(0, dl1_container.fields[key].unit) dl1_container.width = u.Quantity(np.nan, u.m) dl1_container.length = u.Quantity(np.nan, u.m) dl1_container.wl = u.Quantity(np.nan, u.m) for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def get_dl1( calibrated_event, subarray, telescope_id, dl1_container=None, custom_config={}, ): """ Return a DL1ParametersContainer of extracted features from a calibrated event. The DL1ParametersContainer can be passed to be filled if created outside the function (faster for multiple event processing) Parameters ---------- calibrated_event: ctapipe event container subarray: `ctapipe.instrument.subarray.SubarrayDescription` telescope_id: `int` dl1_container: DL1ParametersContainer custom_config: path to a configuration file configuration used for tailcut cleaning superseeds the standard configuration Returns ------- DL1ParametersContainer """ config = replace_config(standard_config, custom_config) cleaning_parameters = config["tailcut"] cleaning_parameters_for_tailcuts = cleaning_parameters.copy() use_main_island = True if "use_only_main_island" in cleaning_parameters.keys(): use_main_island = cleaning_parameters["use_only_main_island"] # time constraint for image cleaning: require at least one neighbor # within delta_time: delta_time = None if "delta_time" in cleaning_parameters: delta_time = cleaning_parameters["delta_time"] # we use pop because ctapipe won't recognize that keyword in tailcuts cleaning_parameters_for_tailcuts.pop("delta_time") cleaning_parameters_for_tailcuts.pop("use_only_main_island") dl1_container = DL1ParametersContainer( ) if dl1_container is None else dl1_container dl1 = calibrated_event.dl1.tel[telescope_id] telescope = subarray.tel[telescope_id] camera_geometry = telescope.camera.geometry image = dl1.image peak_time = dl1.peak_time signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters_for_tailcuts) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: # check the number of islands num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels) if use_main_island: n_pixels_on_island = np.bincount(island_labels) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False if delta_time is not None: cleaned_pixel_times = peak_time # makes sure only signal pixels are used in the time # check: cleaned_pixel_times[~signal_pixels] = np.nan new_mask = apply_time_delta_cleaning(camera_geometry, signal_pixels, cleaned_pixel_times, 1, delta_time) signal_pixels = new_mask # count surviving pixels n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: hillas = hillas_parameters(camera_geometry[signal_pixels], image[signal_pixels]) # Fill container dl1_container.fill_hillas(hillas) # convert ctapipe's width and length (in m) to deg: foclen = subarray.tel[telescope_id].optics.equivalent_focal_length width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.set_timing_features(camera_geometry[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geometry, image, signal_pixels) dl1_container.set_concentration(camera_geometry, image, hillas) dl1_container.n_pixels = n_pixels dl1_container.n_islands = num_islands dl1_container.log_intensity = np.log10(dl1_container.intensity) # We set other fields which still make sense for a non-parametrized # image: dl1_container.set_telescope_info(subarray, telescope_id) return dl1_container
def get_dl1( calibrated_event, subarray, telescope_id, dl1_container=None, custom_config={}, use_main_island=True, ): """ Return a DL1ParametersContainer of extracted features from a calibrated event. The DL1ParametersContainer can be passed to be filled if created outside the function (faster for multiple event processing) Parameters ---------- calibrated_event: ctapipe event container subarray: `ctapipe.instrument.subarray.SubarrayDescription` telescope_id: `int` dl1_container: DL1ParametersContainer custom_config: path to a configuration file configuration used for tailcut cleaning superseeds the standard configuration use_main_island: `bool` Use only the main island to calculate DL1 parameters Returns ------- DL1ParametersContainer """ config = replace_config(standard_config, custom_config) cleaning_parameters = config["tailcut"] dl1_container = DL1ParametersContainer( ) if dl1_container is None else dl1_container dl1 = calibrated_event.dl1.tel[telescope_id] telescope = subarray.tel[telescope_id] camera_geometry = telescope.camera.geometry image = dl1.image peak_time = dl1.peak_time signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: # check the number of islands num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels) if use_main_island: n_pixels_on_island = np.bincount(island_labels.astype(np.int)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geometry[signal_pixels], image[signal_pixels]) # Fill container dl1_container.fill_hillas(hillas) # convert ctapipe's width and length (in m) to deg: foclen = subarray.tel[telescope_id].optics.equivalent_focal_length width = np.rad2deg(np.arctan2(dl1_container.width, foclen)) length = np.rad2deg(np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.set_timing_features(camera_geometry[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geometry, image, signal_pixels) dl1_container.set_concentration(camera_geometry, image, hillas) dl1_container.n_pixels = n_pixels dl1_container.n_islands = num_islands dl1_container.set_telescope_info(subarray, telescope_id) dl1_container.log_intensity = np.log10(dl1_container.intensity) else: # We set other fields which still make sense for a non-parametrized # image: dl1_container.set_telescope_info(subarray, telescope_id) return dl1_container
clean = tailcuts_clean( geom, image, boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors, ) # ignore images with less than 5 pixels after cleaning if clean.sum() < 5: continue # image parameters hillas_c = hillas_parameters(geom[clean], image[clean]) leakage_c = leakage(geom, image, clean) n_islands, island_ids = number_of_islands(geom, clean) timing_c = timing_parameters(geom[clean], image[clean], peakpos[clean], hillas_c) # store parameters for stereo reconstruction hillas_containers[telescope_id] = hillas_c # store timegradients for plotting # ASTRI has no timing in PROD3b, so we use skewness instead if geom.camera_name != "ASTRICam": time_gradients[telescope_id] = timing_c.slope.value else: time_gradients[telescope_id] = hillas_c.skewness print(geom.camera_name, time_gradients[telescope_id])
def main(): std_config = get_standard_config() log.setLevel(logging.INFO) handler = logging.StreamHandler() logging.getLogger().addHandler(handler) if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config if args.pedestal_cleaning: print("Pedestal cleaning") clean_method_name = 'tailcuts_clean_with_pedestal_threshold' sigma = config[clean_method_name]['sigma'] pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma) cleaning_params = get_cleaning_parameters(config, clean_method_name) pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info( f"Fraction of pixel cleaning thresholds above picture thr.:" f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}") picture_th = np.clip(pedestal_thresh, pic_th, None) log.info(f"Tailcut clean with pedestal threshold config used:" f"{config['tailcuts_clean_with_pedestal_threshold']}") else: clean_method_name = 'tailcut' cleaning_params = get_cleaning_parameters(config, clean_method_name) picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info(f"Tailcut config used: {config['tailcut']}") use_only_main_island = True if "use_only_main_island" in config[clean_method_name]: use_only_main_island = config[clean_method_name][ "use_only_main_island"] delta_time = None if "delta_time" in config[clean_method_name]: delta_time = config[clean_method_name]["delta_time"] foclen = OpticsDescription.from_name('LST').equivalent_focal_length cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam") camera_geom = CameraGeometry.from_table(cam_table) dl1_container = DL1ParametersContainer() parameters_to_update = [ 'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi', 'skewness', 'kurtosis', 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'log_intensity' ] nodes_keys = get_dataset_keys(args.input_file) if args.noimage: nodes_keys.remove(dl1_images_lstcam_key) auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys) with tables.open_file(args.input_file, mode='r') as input: image_table = input.root[dl1_images_lstcam_key] dl1_params_input = input.root[dl1_params_lstcam_key].colnames disp_params = { 'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign' } if set(dl1_params_input).intersection(disp_params): parameters_to_update.extend(disp_params) with tables.open_file(args.output_file, mode='a') as output: params = output.root[dl1_params_lstcam_key].read() for ii, row in enumerate(image_table): dl1_container.reset() image = row['image'] peak_time = row['peak_time'] signal_pixels = tailcuts_clean(camera_geom, image, picture_th, boundary_th, isolated_pixels, min_n_neighbors) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: num_islands, island_labels = number_of_islands( camera_geom, signal_pixels) n_pixels_on_island = np.bincount( island_labels.astype(np.int64)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) if use_only_main_island: signal_pixels[ island_labels != max_island_label] = False # if delta_time has been set, we require at least one # neighbor within delta_time to accept a pixel in the image: if delta_time is not None: cleaned_pixel_times = peak_time # makes sure only signal pixels are used in the time # check: cleaned_pixel_times[~signal_pixels] = np.nan new_mask = apply_time_delta_cleaning( camera_geom, signal_pixels, cleaned_pixel_times, 1, delta_time) signal_pixels = new_mask # count the surviving pixels n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels]) dl1_container.fill_hillas(hillas) dl1_container.set_timing_features( camera_geom[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geom, image, signal_pixels) dl1_container.set_concentration( camera_geom, image, hillas) dl1_container.n_islands = num_islands dl1_container.wl = dl1_container.width / dl1_container.length dl1_container.n_pixels = n_pixels width = np.rad2deg( np.arctan2(dl1_container.width, foclen)) length = np.rad2deg( np.arctan2(dl1_container.length, foclen)) dl1_container.width = width dl1_container.length = length dl1_container.log_intensity = np.log10( dl1_container.intensity) if set(dl1_params_input).intersection(disp_params): disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp( dl1_container['x'].to_value(u.m), dl1_container['y'].to_value(u.m), params['src_x'][ii], params['src_y'][ii]) dl1_container['disp_dx'] = disp_dx dl1_container['disp_dy'] = disp_dy dl1_container['disp_norm'] = disp_norm dl1_container['disp_angle'] = disp_angle dl1_container['disp_sign'] = disp_sign for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value output.root[dl1_params_lstcam_key][:] = params
def get_dl1( calibrated_event, subarray, telescope_id, dl1_container=None, custom_config={}, use_main_island=True, ): """ Return a DL1ParametersContainer of extracted features from a calibrated event. The DL1ParametersContainer can be passed to be filled if created outside the function (faster for multiple event processing) Parameters ---------- calibrated_event: ctapipe event container subarray: `ctapipe.instrument.subarray.SubarrayDescription` telescope_id: `int` dl1_container: DL1ParametersContainer custom_config: path to a configuration file configuration used for tailcut cleaning superseeds the standard configuration use_main_island: `bool` Use only the main island to calculate DL1 parameters Returns ------- DL1ParametersContainer """ config = replace_config(standard_config, custom_config) cleaning_parameters = config["tailcut"] dl1_container = DL1ParametersContainer() if dl1_container is None else dl1_container dl1 = calibrated_event.dl1.tel[telescope_id] telescope = subarray.tel[telescope_id] camera_geometry = telescope.camera.geometry image = dl1.image peak_time = dl1.peak_time signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: # check the number of islands num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels) if use_main_island: n_pixels_on_island = np.bincount(island_labels.astype(np.int)) n_pixels_on_island[0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False hillas = hillas_parameters(camera_geometry[signal_pixels], image[signal_pixels]) # Fill container dl1_container.fill_hillas(hillas) dl1_container.set_mc_core_distance(calibrated_event, subarray.positions[telescope_id]) dl1_container.set_mc_type(calibrated_event) dl1_container.set_timing_features(camera_geometry[signal_pixels], image[signal_pixels], peak_time[signal_pixels], hillas) dl1_container.set_leakage(camera_geometry, image, signal_pixels) dl1_container.set_concentration(camera_geometry, image, hillas) dl1_container.n_pixels = n_pixels dl1_container.n_islands = num_islands dl1_container.set_telescope_info(subarray, telescope_id) else: # No image was parametrized, so we put zeros (instead of the default # Nones) in all parameters: a container reset() is not an option because # the default None values prevent the container to be written out. We # cannot use np.nan either, because upon writing it complains for the # integer parameters. # for key in dl1_container.keys(): dl1_container[key] = u.Quantity(0, dl1_container.fields[key].unit) # Fields width and length do not have in their declaration the units # that are actually expected later in the program, so we set them here. # We now use nans since these are floats, and will be later used to # calculate W/L... dl1_container.width = u.Quantity(np.nan, u.m) dl1_container.length = u.Quantity(np.nan, u.m) # We set other fields which still make sense for a non-parametrized # image: dl1_container.set_telescope_info(subarray, telescope_id) return dl1_container