def get_dl1b_tailcut(dl1a_img, dl1a_pulse, config_path, use_main_island=True):
cleaning_method = tailcuts_clean
config = read_configuration_file(config_path)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer()
image = dl1a_img
pulse_time = dl1a_pulse
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
pulse_time[signal_pixels], hillas)
set_converted_hillas_param(dl1_container, dl1_container.width,
dl1_container.length)
set_image_param(dl1_container, image, signal_pixels, hillas, n_pixels,
num_islands)
return dl1_container
def set_n_islands(self, geom, clean):
n_islands, islands_mask = number_of_islands(geom, clean)
self.n_islands = n_islands
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
log.info(f"Tailcut config used: {config['tailcut']}")
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'log_intensity'
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geom[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels],
hillas)
dl1_container.set_leakage(camera_geom, image, signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m),
params['src_x'][ii],
params['src_y'][ii]
)
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'r',
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii % 10000 == 0:
print(ii)
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image,
**config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.r = np.sqrt(dl1_container.x**2 +
dl1_container.y**2)
else:
# for consistency with r0_to_dl1.py:
for key in dl1_container.keys():
dl1_container[key] = \
u.Quantity(0, dl1_container.fields[key].unit)
dl1_container.width = u.Quantity(np.nan, u.m)
dl1_container.length = u.Quantity(np.nan, u.m)
dl1_container.wl = u.Quantity(np.nan, u.m)
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
cleaning_parameters_for_tailcuts = cleaning_parameters.copy()
use_main_island = True
if "use_only_main_island" in cleaning_parameters.keys():
use_main_island = cleaning_parameters["use_only_main_island"]
# time constraint for image cleaning: require at least one neighbor
# within delta_time:
delta_time = None
if "delta_time" in cleaning_parameters:
delta_time = cleaning_parameters["delta_time"]
# we use pop because ctapipe won't recognize that keyword in tailcuts
cleaning_parameters_for_tailcuts.pop("delta_time")
cleaning_parameters_for_tailcuts.pop("use_only_main_island")
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters_for_tailcuts)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels)
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(camera_geometry,
signal_pixels,
cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
# count surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
# Fill container
dl1_container.fill_hillas(hillas)
# convert ctapipe's width and length (in m) to deg:
foclen = subarray.tel[telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(camera_geometry, image, hillas)
dl1_container.n_pixels = n_pixels
dl1_container.n_islands = num_islands
dl1_container.log_intensity = np.log10(dl1_container.intensity)
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
return dl1_container
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
use_main_island=True,
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
use_main_island: `bool` Use only the main island
to calculate DL1 parameters
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
# Fill container
dl1_container.fill_hillas(hillas)
# convert ctapipe's width and length (in m) to deg:
foclen = subarray.tel[telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(camera_geometry, image, hillas)
dl1_container.n_pixels = n_pixels
dl1_container.n_islands = num_islands
dl1_container.set_telescope_info(subarray, telescope_id)
dl1_container.log_intensity = np.log10(dl1_container.intensity)
else:
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
return dl1_container
clean = tailcuts_clean(
geom,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors,
)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(geom[clean], image[clean])
leakage_c = leakage(geom, image, clean)
n_islands, island_ids = number_of_islands(geom, clean)
timing_c = timing_parameters(geom[clean], image[clean], peakpos[clean],
hillas_c)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if geom.camera_name != "ASTRICam":
time_gradients[telescope_id] = timing_c.slope.value
else:
time_gradients[telescope_id] = hillas_c.skewness
print(geom.camera_name, time_gradients[telescope_id])
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
if args.pedestal_cleaning:
print("Pedestal cleaning")
clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
sigma = config[clean_method_name]['sigma']
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(
f"Fraction of pixel cleaning thresholds above picture thr.:"
f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")
else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Tailcut config used: {config['tailcut']}")
use_only_main_island = True
if "use_only_main_island" in config[clean_method_name]:
use_only_main_island = config[clean_method_name][
"use_only_main_island"]
delta_time = None
if "delta_time" in config[clean_method_name]:
delta_time = config[clean_method_name]["delta_time"]
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = [
'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
'concentration_pixel', 'leakage_intensity_width_1',
'leakage_intensity_width_2', 'leakage_pixels_width_1',
'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
'n_pixels', 'wl', 'log_intensity'
]
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {
'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
boundary_th, isolated_pixels,
min_n_neighbors)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int64))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
if use_only_main_island:
signal_pixels[
island_labels != max_island_label] = False
# if delta_time has been set, we require at least one
# neighbor within delta_time to accept a pixel in the image:
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(
camera_geom, signal_pixels, cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
# count the surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(
camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(
np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m), params['src_x'][ii],
params['src_y'][ii])
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
use_main_island=True,
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
use_main_island: `bool` Use only the main island
to calculate DL1 parameters
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer() if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geometry[signal_pixels], image[signal_pixels])
# Fill container
dl1_container.fill_hillas(hillas)
dl1_container.set_mc_core_distance(calibrated_event, subarray.positions[telescope_id])
dl1_container.set_mc_type(calibrated_event)
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels],
hillas)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(camera_geometry, image, hillas)
dl1_container.n_pixels = n_pixels
dl1_container.n_islands = num_islands
dl1_container.set_telescope_info(subarray, telescope_id)
else:
# No image was parametrized, so we put zeros (instead of the default
# Nones) in all parameters: a container reset() is not an option because
# the default None values prevent the container to be written out. We
# cannot use np.nan either, because upon writing it complains for the
# integer parameters.
#
for key in dl1_container.keys():
dl1_container[key] = u.Quantity(0, dl1_container.fields[key].unit)
# Fields width and length do not have in their declaration the units
# that are actually expected later in the program, so we set them here.
# We now use nans since these are floats, and will be later used to
# calculate W/L...
dl1_container.width = u.Quantity(np.nan, u.m)
dl1_container.length = u.Quantity(np.nan, u.m)
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
return dl1_container
