def start(self):
self.log.info('Loop on events...')
for event in tqdm(
self.event_source,
desc='EventWriter',
total=self.event_source.max_events,
disable=~self.progress):
self.event_cutflow.count('no_sel')
self.calibrator.calibrate(event)
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count('no_sel')
camera = event.inst.subarray.tel[tel_id].camera
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image[0] # Waiting for automatic gain selection
mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Preselection cuts
if self.image_cutflow.cut('n_pixel', cleaned):
continue
if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
continue
# Image parametrisation
params = hillas_parameters(camera, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(camera.cam_id, [event.r0, event.mc, params])
def start(self):
self.log.info('Loop on events...')
for event in tqdm(
self.event_source,
desc='EventWriter',
total=self.event_source.max_events,
disable=~self.progress):
self.event_cutflow.count('no_sel')
self.calibrator.calibrate(event)
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count('no_sel')
camera = event.inst.subarray.tel[tel_id].camera
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image[0] # Waiting for automatic gain selection
mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Preselection cuts
if self.image_cutflow.cut('n_pixel', cleaned):
continue
if self.image_cutflow.cut('image_amplitude', np.sum(cleaned)):
continue
# Image parametrisation
params = hillas_parameters(camera, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(camera.cam_id, [event.r0, event.mc, params])
def parametrize_image(image, peak_time, signal_pixels, camera_geometry,
focal_length, dl1_container):
'''
Calculate image parameters and fill them into ``dl1_container``
'''
geom_selected = camera_geometry[signal_pixels]
image_selectecd = image[signal_pixels]
hillas = hillas_parameters(geom_selected, image_selectecd)
# Fill container
dl1_container.fill_hillas(hillas)
# convert ctapipe's width and length (in m) to deg:
for key in ['width', 'width_uncertainty', 'length', 'length_uncertainty']:
value = getattr(dl1_container, key)
setattr(dl1_container, key,
_camera_distance_to_angle(value, focal_length))
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.set_timing_features(
geom_selected,
image_selectecd,
peak_time[signal_pixels],
hillas,
)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(geom_selected, image_selectecd, hillas)
dl1_container.log_intensity = np.log10(dl1_container.intensity)
def start(self):
self.log.info("Loop on events...")
for event in tqdm(
self.event_source,
desc="EventWriter",
total=self.event_source.max_events,
disable=~self.progress,
):
self.calibrator(event)
for tel_id in event.dl0.tel.keys():
geom = self.event_source.subarray.tel[tel_id].camera.geometry
dl1_tel = event.dl1.tel[tel_id]
# Image cleaning
image = dl1_tel.image # Waiting for automatic gain selection
mask = tailcuts_clean(geom,
image,
picture_thresh=10,
boundary_thresh=5)
cleaned = image.copy()
cleaned[~mask] = 0
# Image parametrisation
params = hillas_parameters(geom, cleaned)
# Save Ids, MC infos and Hillas informations
self.writer.write(geom.camera_name,
[event.r0, event.simulation.shower, params])
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
geom = CameraGeometry.from_name('LSTCam-002')
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend(['wl', 'r', 'leakage', 'n_islands', 'intercept', 'time_gradient'])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii%10000 == 0:
print(ii)
image = row['image']
pulse_time = row['pulse_time']
signal_pixels = tailcuts_clean(geom, image, **config['tailcut'])
if image[signal_pixels].shape[0] > 0:
num_islands, island_labels = number_of_islands(geom, signal_pixels)
hillas = hillas_parameters(geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(geom[signal_pixels],
image[signal_pixels],
pulse_time[signal_pixels],
hillas)
dl1_container.set_leakage(geom, image, signal_pixels)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2)
for p in parameters_to_update:
params[ii][p] = Quantity(dl1_container[p]).value
else:
for p in parameters_to_update:
params[ii][p] = 0
output.root[dl1_params_lstcam_key][:] = params
def waveform_loop(self, file_reader):
global N_EVENTS
global LIVE_DATA
global FREEZE_DATA
# Open file
# N_EVENTS = file_reader.num_events
with tqdm() as pbar:
source = file_reader.read()
for event in source:
# TODO: Remove telid loop once hillas tested
for telid in event.r0.tels_with_data:
pbar.update(1)
self.r1.calibrate(event)
self.dl0.reduce(event)
self.dl1.calibrate(event)
index = event.count
image = event.dl1.tel[telid].image[CHAN]
mask = tailcuts_clean(self.geom, image, 1, 8, 5)
cleaned = np.ma.masked_array(image, ~mask)
pos = event.inst.pixel_pos[telid]
try:
hillas = hillas_parameters(*pos, cleaned)
# hillas = hillas_parameters_2(*pos, cleaned)
except HillasParameterizationError:
print('HillasParameterizationError')
continue
# print(hillas[0].length)
live_d = dict(index=index, image=intensity_to_hex(cleaned))
live_d = dict(index=index, image=intensity_to_hex(image))
LIVE_DATA = live_d
freeze_d = dict(index=index, event=copy.deepcopy(event))
FREEZE_DATA = freeze_d
width = hillas.width
length = hillas.length
size = hillas.size
phi = hillas.phi
miss = hillas.miss
r = hillas.r
HILLAS['width'] += \
np.histogram(width, bins=100, range=[0, 1])[0]
HILLAS['length'] += \
np.histogram(length, bins=100, range=[0, 1])[0]
HILLAS['size'] += \
np.histogram(size, bins=100, range=[0, 1000])[0]
HILLAS['phi'] += \
np.histogram(phi, bins=100, range=[0, 4])[0]
HILLAS['miss'] += \
np.histogram(miss, bins=100, range=[0, 1])[0]
HILLAS['r'] += \
np.histogram(r, bins=100, range=[0, 1])[0]
def get_dl1(calibrated_event,
telescope_id,
dl1_container=None,
custom_config={}):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
event: ctapipe event container
telescope_id: int
dl1_container: DL1ParametersContainer
config_file: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
tel = calibrated_event.inst.subarray.tels[telescope_id]
dl1 = calibrated_event.dl1.tel[telescope_id]
camera = tel.camera
image = dl1.image
pulse_time = dl1.pulse_time
signal_pixels = cleaning_method(camera, image, **cleaning_parameters)
if image[signal_pixels].sum() > 0:
hillas = hillas_parameters(camera[signal_pixels], image[signal_pixels])
# Fill container
dl1_container.fill_mc(calibrated_event)
dl1_container.fill_hillas(hillas)
dl1_container.fill_event_info(calibrated_event)
dl1_container.set_mc_core_distance(calibrated_event, telescope_id)
dl1_container.set_mc_type(calibrated_event)
dl1_container.set_timing_features(camera, image, pulse_time, hillas)
dl1_container.set_leakage(camera, image, signal_pixels)
dl1_container.set_n_islands(camera, signal_pixels)
dl1_container.set_source_camera_position(calibrated_event,
telescope_id)
dl1_container.set_disp([dl1_container.src_x, dl1_container.src_y],
hillas)
dl1_container.set_telescope_info(calibrated_event, telescope_id)
return dl1_container
else:
return None
def process_event(event, dl1, infos_save):
Keylist = [
"size", "cen_x", "cen_y", "length", "width", "r", "phi", "psi", "miss",
"skewness", "kurtosis", "Reinheit", "Effizienz", "Genauigkeit"
]
calibration(event, r1, dl0, dl1)
eins_drinne = {}
err = False
image = event.dl1.tel[tel_id].image[0]
geom = event.inst.subarray.tel[tel_id].camera
for i in infos_save.keys():
for j in infos_save[i].keys():
if i not in eins_drinne.keys():
eins_drinne[i] = {}
if j not in eins_drinne[i].keys():
eins_drinne[i][j] = False
if i == "o":
event_info = {}
event_info = set_mc(event_info, event, tel_id)
infos_save[i][j].append(event_info)
eins_drinne[i][j] = True
continue
cleaning_mask = tailcuts_clean(geom,
image,
picture_thresh=i,
boundary_thresh=j)
if len(image[cleaning_mask]) == 0:
continue
clean = image.copy()
clean[~cleaning_mask] = 0.0
event_info = {}
hillas = hillas_parameters(geom, image=clean)
event_info = set_hillas(event_info, hillas)
event_info = set_mc(event_info, event, tel_id)
event_info = set_tp(event_info, cleaning_mask,
event.mc.tel[tel_id].photo_electron_image)
for key in Keylist:
if key not in event_info:
err = True
else:
drinne = False
wert = event_info[key][0]
if wert >= 0:
drinne = True
elif wert < 0:
drinne = True
if drinne is False:
err = True
if err:
continue
infos_save[i][j].append(event_info)
eins_drinne[i][j] = True
return infos_save
def get_observation_parameters(charge: np.array,
peak: np.array,
cam_name: str,
cutflow: CutFlow,
boundary_threshold: float = None,
picture_threshold: float = None,
min_neighbours: float = None,
plot: bool = False,
cut: bool = True):
"""
:param charge: Charge image
:param peak: Peak time image
:param cam_name: Camera name. e.g. FlashCam, ASTRICam, etc.
:param cutflow: Cutflow for selection
:param boundary_threshold: (Optional) Cleaning parameter: boundary threshold
:param picture_threshold: (Optional) Cleaning parameter: picture threshold
:param min_neighbours: (Optional) Cleaning parameter: minimum neighbours
:param plot: If True, for each observation a plot will be shown (Default: False)
:param cut: If true, tight else loose
:return: hillas containers, leakage container, number of islands, island IDs, timing container, timing gradient
"""
charge_biggest, mask = clean_charge(charge, cam_name, boundary_threshold,
picture_threshold, min_neighbours)
camera = get_camera(cam_name)
geometry = camera.geometry
charge_biggest, camera_biggest, n_islands = mask_from_biggest_island(
charge, geometry, mask)
if cut:
if cutflow.cut(CFO_MIN_PIXEL, charge_biggest):
return
if cutflow.cut(CFO_MIN_CHARGE, np.sum(charge_biggest)):
return
if cutflow.cut(CFO_NEGATIVE_CHARGE, charge_biggest):
return
leakage_c = leakage(geometry, charge, mask)
if plot:
_, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
CameraDisplay(geometry, charge, ax=ax1).add_colorbar()
CameraDisplay(camera_biggest, charge_biggest, ax=ax2).add_colorbar()
plt.show()
moments = hillas_parameters(camera_biggest, charge_biggest)
if cut:
if cutflow.cut(CFO_CLOSE_EDGE, moments, camera.camera_name):
return
if cutflow.cut(CFO_BAD_ELLIP, moments):
return
if cutflow.cut(CFO_POOR_MOMENTS, moments):
return
timing_c = timing_parameters(geometry, charge, peak, moments, mask)
time_gradient = timing_c.slope.value if geometry.camera_name != 'ASTRICam' else moments.skewness
return moments, leakage_c, timing_c, time_gradient, n_islands
def test_overlay_disp_vector():
from ctapipe.image import hillas_parameters
geom = CameraGeometry.from_name('LSTCam')
image = np.random.rand(geom.n_pixels)
display = CameraDisplay(geom, image)
hillas = hillas_parameters(geom, image)
disp = disp_parameters_event(hillas, 0.1 * u.m, 0.3 * u.m)
overlay_disp_vector(display, disp, hillas)
def get_dl1(calibrated_event, telescope_id, dl1_container=None):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
event: ctapipe event container
telescope_id: int
dl1_container: DL1ParametersContainer
Returns
-------
DL1ParametersContainer
"""
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
tel = calibrated_event.inst.subarray.tels[telescope_id]
dl1 = calibrated_event.dl1.tel[telescope_id]
camera = tel.camera
waveform = calibrated_event.r0.tel[telescope_id].waveform
image = dl1.image
pulse_time = dl1.pulse_time
image, pulse_time = gain_selection(waveform, image, pulse_time,
camera.cam_id, threshold)
signal_pixels = cleaning_method(camera, image, **cleaning_parameters)
image[~signal_pixels] = 0
if image.sum() > 0:
hillas = hillas_parameters(camera, image)
# Fill container
dl1_container.fill_mc(calibrated_event)
dl1_container.fill_hillas(hillas)
dl1_container.fill_event_info(calibrated_event)
dl1_container.set_mc_core_distance(calibrated_event, telescope_id)
dl1_container.set_mc_type(calibrated_event)
dl1_container.set_timing_features(camera, image, pulse_time, hillas)
dl1_container.set_leakage(camera, image, signal_pixels)
dl1_container.set_n_islands(camera, signal_pixels)
dl1_container.set_source_camera_position(calibrated_event,
telescope_id)
dl1_container.set_disp([dl1_container.src_x, dl1_container.src_y],
hillas)
dl1_container.set_telescope_info(calibrated_event, telescope_id)
return dl1_container
else:
return None
def compute_hillas_parameters(events, geom):
for event in events:
mask = event.data.cleaning_mask
image = event.data.reconstructed_number_of_pe.copy()
image[image < 0] = 0
image[~mask] = 0
try:
hillas = hillas_parameters(geom, image)
event.hillas = hillas
yield event
except HillasParameterizationError:
continue
def get_dl1(calibrated_event, telescope_id):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event
Parameters
----------
event: ctapipe event container
telescope_id:
Returns
-------
DL1ParametersContainer
"""
dl1_container = DL1ParametersContainer()
tel = calibrated_event.inst.subarray.tels[telescope_id]
dl1 = calibrated_event.dl1.tel[telescope_id]
camera = tel.camera
waveform = calibrated_event.r0.tel[telescope_id].waveform
image = dl1.image
peakpos = dl1.peakpos
image, peakpos = gain_selection(waveform, image, peakpos, camera.cam_id,
threshold)
signal_pixels = cleaning_method(camera, image, **cleaning_parameters)
image[~signal_pixels] = 0
if image.sum() > 0:
try:
hillas = hillas_parameters(camera, image)
## Fill container ##
dl1_container.fill_mc(calibrated_event)
dl1_container.fill_hillas(hillas)
dl1_container.fill_event_info(calibrated_event)
dl1_container.set_mc_core_distance(calibrated_event, telescope_id)
# dl1_container.mc_type = utils.guess_type(infile)
dl1_container.set_timing_features(camera, image, peakpos, hillas)
dl1_container.set_source_camera_position(calibrated_event,
telescope_id)
dl1_container.set_disp([dl1_container.src_x, dl1_container.src_y],
hillas)
return dl1_container
except:
print("Bad event")
return None
else:
return None
def draw_several_cams(geom, ncams=4):
cmaps = ["jet", "afmhot", "terrain", "autumn"]
fig, axs = plt.subplots(
1,
ncams,
figsize=(15, 4),
)
for ii in range(ncams):
disp = CameraDisplay(
geom,
ax=axs[ii],
title="CT{}".format(ii + 1),
)
disp.cmap = cmaps[ii]
model = toymodel.Gaussian(
x=(0.2 - ii * 0.1) * u.m,
y=(-ii * 0.05) * u.m,
width=(0.05 + 0.001 * ii) * u.m,
length=(0.15 + 0.05 * ii) * u.m,
psi=ii * 20 * u.deg,
)
image, _, _ = model.generate_image(
geom,
intensity=1500,
nsb_level_pe=5,
)
mask = tailcuts_clean(
geom,
image,
picture_thresh=6 * image.mean(),
boundary_thresh=4 * image.mean(),
)
cleaned = image.copy()
cleaned[~mask] = 0
hillas = hillas_parameters(geom, cleaned)
disp.image = image
disp.add_colorbar(ax=axs[ii])
disp.set_limits_percent(95)
disp.overlay_moments(hillas, linewidth=3, color="blue")
def draw_several_cams(geom, ncams=4):
cmaps = ['jet', 'afmhot', 'terrain', 'autumn']
fig, axs = plt.subplots(
1, ncams, figsize=(15, 4), sharey=True, sharex=True
)
for ii in range(ncams):
disp = CameraDisplay(
geom,
ax=axs[ii],
title="CT{}".format(ii + 1),
)
disp.cmap = cmaps[ii]
model = toymodel.generate_2d_shower_model(
centroid=(0.2 - ii * 0.1, -ii * 0.05),
width=0.005 + 0.001 * ii,
length=0.1 + 0.05 * ii,
psi=ii * 20 * u.deg,
)
image, sig, bg = toymodel.make_toymodel_shower_image(
geom,
model.pdf,
intensity=50,
nsb_level_pe=1000,
)
mask = tailcuts_clean(
geom,
image,
picture_thresh=6 * image.mean(),
boundary_thresh=4 * image.mean()
)
cleaned = image.copy()
cleaned[~mask] = 0
hillas = hillas_parameters(geom, cleaned)
disp.image = image
disp.add_colorbar(ax=axs[ii])
disp.set_limits_percent(95)
disp.overlay_moments(hillas, linewidth=3, color='blue')
def draw_several_cams(geom, ncams=4):
cmaps = ['jet', 'afmhot', 'terrain', 'autumn']
fig, axs = plt.subplots(
1, ncams, figsize=(15, 4),
)
for ii in range(ncams):
disp = CameraDisplay(
geom,
ax=axs[ii],
title="CT{}".format(ii + 1),
)
disp.cmap = cmaps[ii]
model = toymodel.generate_2d_shower_model(
centroid=(0.2 - ii * 0.1, -ii * 0.05),
width=0.05 + 0.001 * ii,
length=0.15 + 0.05 * ii,
psi=ii * 20 * u.deg,
)
image, sig, bg = toymodel.make_toymodel_shower_image(
geom,
model.pdf,
intensity=1500,
nsb_level_pe=5,
)
mask = tailcuts_clean(
geom,
image,
picture_thresh=6 * image.mean(),
boundary_thresh=4 * image.mean()
)
cleaned = image.copy()
cleaned[~mask] = 0
hillas = hillas_parameters(geom, cleaned)
disp.image = image
disp.add_colorbar(ax=axs[ii])
disp.set_limits_percent(95)
disp.overlay_moments(hillas, linewidth=3, color='blue')
def compute_hillas_parameters(events, geom):
for event in events:
mask = event.data.cleaning_mask
image = event.data.reconstructed_number_of_pe
image = np.ma.masked_array(image, mask=~mask)
try:
hillas = hillas_parameters(geom, image)
event.hillas = hillas
yield event
except HillasParameterizationError:
continue
def get_dl1b_tailcut(dl1a_img, dl1a_pulse, config_path, use_main_island=True):
cleaning_method = tailcuts_clean
config = read_configuration_file(config_path)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer()
image = dl1a_img
pulse_time = dl1a_pulse
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
pulse_time[signal_pixels], hillas)
set_converted_hillas_param(dl1_container, dl1_container.width,
dl1_container.length)
set_image_param(dl1_container, image, signal_pixels, hillas, n_pixels,
num_islands)
return dl1_container
def find_nice_event():
filename = datasets.get_dataset_path("gamma_test_large.simtel.gz")
source = event_source(filename)
calib = CameraCalibrator()
for i, event in tqdm(enumerate(source)):
# from IPython import embed; embed()
# if i in [0, 1, 2, 3, 4, 39]: # skip ugly events
# print(i, event.mc.energy)
# continue
subarray = event.inst.subarray
calib(event)
for tel_id in event.dl0.tels_with_data:
# Camera Geometry required for hillas parametrization
camgeom = subarray.tel[tel_id].camera
# note the [0] is for channel 0 which is high-gain channel
image = event.dl1.tel[tel_id].image
# Cleaning of the image
cleaned_image = image.copy()
# create a clean mask of pixels above the threshold
cleanmask = tailcuts_clean(
camgeom, image, picture_thresh=10, boundary_thresh=5, min_number_picture_neighbors=3
)
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calculate hillas parameters
try:
d = hillas_parameters(camgeom, cleaned_image)
except HillasParameterizationError:
pass # skip failed parameterization (normally no signal)
# from IPython import embed; embed()
tel_name = event.inst.subarray.tel[tel_id].name
if tel_name == 'LST' and d.r < 1 * u.m and d.intensity > 400:
print(i, d.intensity, event.mc.energy)
return tel_id, d, event, cleanmask
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'r',
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii % 10000 == 0:
print(ii)
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image,
**config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.r = np.sqrt(dl1_container.x**2 +
dl1_container.y**2)
else:
# for consistency with r0_to_dl1.py:
for key in dl1_container.keys():
dl1_container[key] = \
u.Quantity(0, dl1_container.fields[key].unit)
dl1_container.width = u.Quantity(np.nan, u.m)
dl1_container.length = u.Quantity(np.nan, u.m)
dl1_container.wl = u.Quantity(np.nan, u.m)
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
cleaned_image = image.copy()
# create a clean mask of pixels above the threshold
cleanmask = tailcuts_clean(camgeom,
image,
picture_thresh=10,
boundary_thresh=5)
if np.count_nonzero(cleanmask) < 10:
continue
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calculate hillas parameters
try:
hillas_dict[tel_id] = hillas_parameters(camgeom, cleaned_image)
except HillasParameterizationError:
continue # skip failed parameterization (normally no signal)
timing_dict[tel_id] = timing_parameters(camgeom, image, time,
hillas_dict[tel_id],
cleanmask).slope.value
array_disp.set_vector_hillas(hillas_dict,
500,
timing_dict,
angle_offset=0 * u.deg)
plt.pause(0.1) # allow matplotlib to redraw the display
if len(hillas_dict) < 2:
if __name__ == "__main__":
# Load the camera
geom = CameraGeometry.from_name("LSTCam")
disp = CameraDisplay(geom)
disp.add_colorbar()
# Create a fake camera image to display:
model = toymodel.Gaussian(
x=0.2 * u.m, y=0.0 * u.m, width=0.05 * u.m, length=0.15 * u.m, psi="35d"
)
image, sig, bg = model.generate_image(geom, intensity=1500, nsb_level_pe=2)
# Apply image cleaning
cleanmask = tailcuts_clean(geom, image, picture_thresh=10, boundary_thresh=5)
clean = image.copy()
clean[~cleanmask] = 0.0
# Calculate image parameters
hillas = hillas_parameters(geom, clean)
print(hillas)
# Show the camera image and overlay Hillas ellipse and clean pixels
disp.image = image
disp.cmap = "inferno"
disp.highlight_pixels(cleanmask, color="crimson")
disp.overlay_moments(hillas, color="cyan", linewidth=1)
plt.show()
# note the [0] is for channel 0 which is high-gain channel
image = event.dl1.tel[tel_id].image
# Cleaning of the image
cleaned_image = image
# create a clean mask of pixels above the threshold
cleanmask = tailcuts_clean(
camgeom, image, picture_thresh=10, boundary_thresh=5
)
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calculate hillas parameters
# It fails for empty pixels
try:
params = hillas_parameters(camgeom, cleaned_image)
except:
continue
if params.width > 0:
hillas_params[tel_id] = params
array_pointing = SkyCoord(
az=event.mcheader.run_array_direction[0],
alt=event.mcheader.run_array_direction[1],
frame=horizon_frame
)
if len(hillas_params) < 2:
if __name__ == '__main__':
# Load the camera
geom = CameraGeometry.from_name("LSTCam")
disp = CameraDisplay(geom)
disp.add_colorbar()
# Create a fake camera image to display:
model = toymodel.generate_2d_shower_model(
centroid=(0.2, 0.0), width=0.05, length=0.15, psi='35d'
)
image, sig, bg = toymodel.make_toymodel_shower_image(
geom, model.pdf, intensity=1500, nsb_level_pe=3
)
# Apply image cleaning
cleanmask = tailcuts_clean(
geom, image, picture_thresh=10, boundary_thresh=5
)
# Calculate image parameters
hillas = hillas_parameters(geom[cleanmask], image[cleanmask])
# Show the camera image and overlay Hillas ellipse and clean pixels
disp.image = image
disp.highlight_pixels(cleanmask, color='crimson')
disp.overlay_moments(hillas, color='cyan', linewidth=3)
plt.show()
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
cleaning_parameters_for_tailcuts = cleaning_parameters.copy()
use_main_island = True
if "use_only_main_island" in cleaning_parameters.keys():
use_main_island = cleaning_parameters["use_only_main_island"]
# time constraint for image cleaning: require at least one neighbor
# within delta_time:
delta_time = None
if "delta_time" in cleaning_parameters:
delta_time = cleaning_parameters["delta_time"]
# we use pop because ctapipe won't recognize that keyword in tailcuts
cleaning_parameters_for_tailcuts.pop("delta_time")
cleaning_parameters_for_tailcuts.pop("use_only_main_island")
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters_for_tailcuts)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels)
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(camera_geometry,
signal_pixels,
cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
# count surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
# Fill container
dl1_container.fill_hillas(hillas)
# convert ctapipe's width and length (in m) to deg:
foclen = subarray.tel[telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(camera_geometry, image, hillas)
dl1_container.n_pixels = n_pixels
dl1_container.n_islands = num_islands
dl1_container.log_intensity = np.log10(dl1_container.intensity)
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
return dl1_container
width=0.01,
length=0.1,
psi='35d')
image, sig, bg = toymodel.make_toymodel_shower_image(geom, model.pdf,
intensity=50,
nsb_level_pe=1000)
# Apply image cleaning
cleanmask = tailcuts_clean(geom, image, picture_thresh=200,
boundary_thresh=100)
clean = image.copy()
clean[~cleanmask] = 0.0
# Calculate image parameters
hillas = hillas_parameters(geom.pix_x, geom.pix_y, clean)
print(hillas)
# Show the camera image and overlay Hillas ellipse and clean pixels
disp.image = image
disp.cmap = 'PuOr'
disp.highlight_pixels(cleanmask, color='black')
disp.overlay_moments(hillas, color='cyan', linewidth=3)
# Draw the neighbors of pixel 100 in red, and the neighbor-neighbors in
# green
for ii in geom.neighbors[130]:
draw_neighbors(geom, ii, color='green')
draw_neighbors(geom, 130, color='cyan', lw=2)
# note the [0] is for channel 0 which is high-gain channel
image = event.dl1.tel[tel_id].image[0]
# Cleaning of the image
cleaned_image = image
# create a clean mask of pixels above the threshold
cleanmask = tailcuts_clean(
camgeom, image, picture_thresh=10, boundary_thresh=5
)
# set all rejected pixels to zero
cleaned_image[~cleanmask] = 0
# Calulate hillas parameters
# It fails for empty pixels
try:
hillas_params[tel_id] = hillas_parameters(camgeom, cleaned_image)
except:
pass
if len(hillas_params) < 2:
continue
reco_result = reco.predict(hillas_params, event.inst, point_altitude, point_azimuth)
# get angular offset between reconstructed shower direction and MC
# generated shower direction
off_angle = angular_separation(
event.mc.az,
event.mc.alt,
reco_result.az,
reco_result.alt
def func(paths, outpath, ro, rc, rn):
# iterate on each proton file & concatenate charge arrays
for n, f in enumerate(paths):
# get the data from the file
try:
print("Opening file #{}...".format(n))
data_p = tables.open_file(f)
_, LST_event_index, LST_image_charge, LST_image_peak_times = get_LST_data(data_p)
_, ei_alt, ei_az, ei_mc_energy = get_event_data(data_p)
# Excluding the 0th element - it's the empty one!!
LST_event_index = LST_event_index[1:]
LST_image_charge = LST_image_charge[1:]
LST_image_peak_times = LST_image_peak_times[1:]
# get camera geometry & camera pixels coordinates
camera = CameraGeometry.from_name("LSTCam")
points = np.array([np.array(camera.pix_x / u.m), np.array(camera.pix_y / u.m)]).T
# original choice by Nicola: 100x100 points in 2.5m x 2.5m
#grid_x, grid_y = np.mgrid[-1.25:1.25:100j, -1.25:1.25:100j]
# I choose instead 96x88 px in 2.40m x 2.20m: same spatial separation, less points
grid_x, grid_y = np.mgrid[-1.20:1.20:96j, -1.10:1.10:88j]
'''
was probably useless and wrong even with the old simulations
if alt_array > 90:
alt_array = 90
'''
# LST coordinates (pointing position)
#point = AltAz(alt=alt_array * u.deg, az=az_array * u.deg)
lst_image_charge_interp = []
lst_image_peak_times_interp = []
# alt az of the array [deg]
#az_array = 180. # before was ai_run_array_direction[0][0], now it is hardcoded as it's not present in new files
#alt_array = 70. # ai_run_array_direction[0][1]
#delta_az = []
#delta_alt = []
intensities = []
intensities_width_2 = []
acc_idxs = [] # accepted indexes # in principle it can be removed when cuts (line AAA) are not used here
cleaning_level = {'LSTCam': (3.5, 7.5, 2)}
count = 0
#rejected = open(f[:-3] + "_rejected.txt", "w")
for i in trange(0, len(LST_image_charge), desc="Images interpolation"):
image = LST_image_charge[i]
time = LST_image_peak_times[i]
boundary, picture, min_neighbors = cleaning_level['LSTCam']
clean = tailcuts_clean(
camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors
)
if len(np.where(clean > 0)[0]) != 0:
hillas = hillas_parameters(camera[clean], image[clean])
intensity = hillas['intensity']
l = leakage(camera, image, clean)
# print(l)
leakage2_intensity = l['intensity_width_2']
# if intensity > 50 and leakage2_intensity < 0.2: # ------>AAA --- CUT DIRECTLY DURING INTERP
# cubic interpolation
interp_img = griddata(points, image, (grid_x, grid_y), fill_value=0, method='cubic')
interp_time = griddata(points, time, (grid_x, grid_y), fill_value=0, method='cubic')
# delta az, delta alt computation
#az = ei_az[LST_event_index[i]]
#alt = ei_alt[LST_event_index[i]]
#src = AltAz(alt=alt * u.rad, az=az * u.rad)
#source_direction = src.transform_to(NominalFrame(origin=point))
# appending to arrays
lst_image_charge_interp.append(interp_img)
lst_image_peak_times_interp.append(interp_time)
#delta_az.append(source_direction.delta_az.deg)
#delta_alt.append(source_direction.delta_alt.deg)
intensities.append(intensity)
intensities_width_2.append(leakage2_intensity)
acc_idxs += [i] # also this one can be removed when no cuts here
else:
count += 1
# #rejected.write("{}.\tImage #{} rejected (no islands)!\n".format(count, i))
# print("No islands: image #{} rejected! (cumulative: {})\n".format(i, count), end='\r')
# lst_image_charge_interp = np.array(lst_image_charge_interp)
print("Number of rejected images: {} ({:.1f}%)".format(count, count / len(intensities) *100))
data_p.close()
#rejected.close()
fpath = Path(f)
newname = fpath.name[:-3] + '_interp.h5'
filename = str(PurePath(outpath, newname))
print("Writing file: " + filename + "\n")
data_file = h5py.File(filename, 'w')
data_file.create_dataset('Event_Info/ei_alt', data=np.array(ei_alt))
data_file.create_dataset('Event_Info/ei_az', data=np.array(ei_az))
data_file.create_dataset('Event_Info/ei_mc_energy', data=np.array(ei_mc_energy))
data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index)[acc_idxs])
data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge)[acc_idxs])
data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times)[acc_idxs])
# data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index))
# data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge))
# data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times))
data_file.create_dataset('LST/LST_image_charge_interp', data=np.array(lst_image_charge_interp))
data_file.create_dataset('LST/LST_image_peak_times_interp', data=np.array(lst_image_peak_times_interp))
#data_file.create_dataset('LST/delta_alt', data=np.array(delta_alt))
#data_file.create_dataset('LST/delta_az', data=np.array(delta_az))
data_file.create_dataset('LST/intensities', data=np.array(intensities))
data_file.create_dataset('LST/intensities_width_2', data=np.array(intensities_width_2))
data_file.close()
# in the interpolated files there will be all the original events
# but for the LST only the ones actually see at least from one LST (as in the original files)
# and that are above thresholds cuts
if ro == '1':
remove(f)
print('Removing original file')
except HDF5ExtError:
print('\nUnable to open file' + f)
if rc == '1':
print('Removing it...')
remove(f)
except NoSuchNodeError:
print('This file has a problem with the data structure: ' + f)
if rn == '1':
print('Removing it...')
remove(f)
def get_events(filename,
storedata=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Read a Simtelarray file, extract pixels charge, calculate image parameters and timing
parameters and store the result in an hdf5 file.
Parameters:
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
"""
#Particle type:
particle_type = guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y',
'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi',
'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel',
'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX',
'SrcY', 'disp', 'hadroness'
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = EventSourceFactory.produce(
input_url=filename, allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
'''
if i > 100: # for quick tests
break
'''
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal
# the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray
nsamples = data.shape[2] # total number of samples
pedcorrectedsamples = data - np.atleast_3d(
ped
) / nsamples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
integrator = LocalPeakIntegrator(None, None)
integration, peakpos, window = integrator.extract_charge(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(
np.log10(signals)
) # This seems to be faster like this, with normal python lists
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(
cleanmask
) # This seems to be faster like this, with normal python lists
clean = signals.copy()
clean[
~cleanmask] = 0.0 # set to 0 pixels which did not survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(
geom,
clean) # this one gives some warnings invalid value in sqrt
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean,
peak_time, hillas.psi)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#Store parameters from event and MC:
ObsID = event.r0.obs_id
EvID = event.r0.event_id
mcEnergy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mcAlt = event.mc.alt.value
mcAz = event.mc.az.value
mcCore_x = event.mc.core_x.value
mcCore_y = event.mc.core_y.value
mcHfirst = event.mc.h_first_int.value
mcType = event.mc.shower_primary_id
mcAztel = event.mcheader.run_array_direction[0].value
mcAlttel = event.mcheader.run_array_direction[1].value
mcXmax = event.mc.x_max.value
GPStime = event.trig.gps_time.value
impact = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
time_gradient = timepars[0].value
intercept = timepars[1].value
#Calculate Disp and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = transformations.calc_CamSourcePos(
mcAlt, mcAz, mcAlttel, mcAztel, focal_length)
SrcX = sourcepos[0]
SrcY = sourcepos[1]
disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x,
y)
hadroness = 0
if particle_type == 'proton':
hadroness = 1
eventdf = pd.DataFrame([[
ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y,
mcHfirst, mcType, GPStime, width, length, width / length,
phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel,
mcAztel, impact, mcXmax, time_gradient, intercept, SrcX,
SrcY, disp, hadroness
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if concatenate == False or (concatenate == True and
np.DataSource().exists(outfile) == False):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
# Create a fake camera image to display:
model = toymodel.Gaussian(
x=0.2 * u.m, y=0.0 * u.m,
width=0.05 * u.m, length=0.15 * u.m,
psi='35d'
)
image, sig, bg = model.generate_image(
geom, intensity=1500, nsb_level_pe=2
)
# Apply image cleaning
cleanmask = tailcuts_clean(
geom, image, picture_thresh=10, boundary_thresh=5
)
clean = image.copy()
clean[~cleanmask] = 0.0
# Calculate image parameters
hillas = hillas_parameters(geom, clean)
print(hillas)
# Show the camera image and overlay Hillas ellipse and clean pixels
disp.image = image
disp.cmap = 'inferno'
disp.highlight_pixels(cleanmask, color='crimson')
disp.overlay_moments(hillas, color='cyan', linewidth=1)
plt.show()
def start(self):
disp = None
for event in tqdm(
self.event_source,
desc=f"Tel{self.tel}",
total=self.event_source.max_events,
disable=~self.progress,
):
self.log.debug(event.trigger)
self.log.debug(f"Energy: {event.simulation.shower.energy}")
self.calibrator(event)
if disp is None:
geom = self.event_source.subarray.tel[self.tel].camera.geometry
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title("CT{:03d} ({}), event {:06d}".format(
self.tel, geom.camera_name, event.index.event_id))
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].waveform
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle(f"Sample {ii:03d}")
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:10d}"
f"_S{ii:02d}.png")
else:
# display integrated event:
im = event.dl1.tel[self.tel].image
if self.clean:
mask = tailcuts_clean(geom,
im,
picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(geom, image=im)
disp.overlay_moments(params,
color="pink",
lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig(
f"CT{self.tel:03d}_EV{event.index.event_id:010d}.png")
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning(
"No events for tel {} were found in {}. Try a "
"different EventIO file or another telescope".format(
self.tel, self.infile))
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
log.info(f"Tailcut config used: {config['tailcut']}")
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'log_intensity'
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geom[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels],
hillas)
dl1_container.set_leakage(camera_geom, image, signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m),
params['src_x'][ii],
params['src_y'][ii]
)
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def update(frame):
centroid = np.random.uniform(-fov, fov, size=2) * scale
width = np.random.uniform(0, maxwid-minwid) * scale + minwid
length = np.random.uniform(0, maxlen) * scale + width
angle = np.random.uniform(0, 360)
intens = np.random.exponential(2) * 500
model = toymodel.generate_2d_shower_model(centroid=centroid,
width=width,
length=length,
psi=angle * u.deg)
self.log.debug(
"Frame=%d width=%03f length=%03f intens=%03d",
frame, width, length, intens
)
image, sig, bg = toymodel.make_toymodel_shower_image(
geom,
model.pdf,
intensity=intens,
nsb_level_pe=3,
)
# alternate between cleaned and raw images
if self._counter == self.cleanframes:
plt.suptitle("Image Cleaning ON")
self.imclean = True
if self._counter == self.cleanframes * 2:
plt.suptitle("Image Cleaning OFF")
self.imclean = False
self._counter = 0
disp.clear_overlays()
if self.imclean:
cleanmask = tailcuts_clean(geom, image,
picture_thresh=10.0,
boundary_thresh=5.0)
for ii in range(2):
dilate(geom, cleanmask)
image[cleanmask == 0] = 0 # zero noise pixels
try:
hillas = hillas_parameters(geom, image)
disp.overlay_moments(hillas, with_label=False,
color='red', alpha=0.7,
linewidth=2, linestyle='dashed')
except HillasParameterizationError:
disp.clear_overlays()
pass
self.log.debug("Frame=%d image_sum=%.3f max=%.3f",
self._counter, image.sum(), image.max())
disp.image = image
if self.autoscale:
disp.set_limits_percent(95)
else:
disp.set_limits_minmax(-5, 200)
disp.axes.figure.canvas.draw()
self._counter += 1
return [ax, ]
# cleaning
boundary, picture, min_neighbors = cleaning_level[camera.cam_id]
clean = tailcuts_clean(
camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors
)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(camera[clean], image[clean])
leakage_c = leakage(camera, image, clean)
n_islands, island_ids = number_of_islands(camera, clean)
timing_c = timing_parameters(
camera[clean], image[clean], peakpos[clean], hillas_c
)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
pointing_azimuth[telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if camera.cam_id != 'ASTRICam':
def iterate_param(possible_cleaning_levels,
event,
camera,
image,
telescope_id,
quality_param_list,
hillas_containers,
index=0):
"""
iteration of the possible cleaning levels and research of the optimal through quality parameters
Parameters
----------
possible_cleaning_levels:
array with all the possible cleaning levels chosen
event:
calibrated event
camera:
Camera geometry information
image:
pixel array that has to be cleaned
telescope_id:
unique number for the identification of a telescope
quality_param_list:
list of all the quality parameters of a cleaned image for all the possible cleaning levels chosen
hillas_containers:
dictionary with telescope IDs as key and
HillasParametersContainer instances as values
index:
index of the current cleaning level chosen for the comparision
Returns
-------
new_param:
the better parameters for the given image
"""
while index < len(possible_cleaning_levels):
picture, boundary, min_neighbors = possible_cleaning_levels[index]
clean = tailcuts_clean(camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors)
# if the size of the clean mask is too little, we can't have a good ellipse reconstruction
flag = verify_size_clean_mask(clean, 6)
if flag:
break
cleaned_image = event.dl1.tel[telescope_id].image.copy()
cleaned_image[~clean] = 0.0
hillas_c = hillas_parameters(camera[clean], image[clean])
hillas_containers[telescope_id] = hillas_c
closest_approach = closest_approach_distance(
hillas_containers[telescope_id])
likelihood = likelihood_function(event, cleaned_image, telescope_id)
num_diff_pixels = numbers_of_different_pixels(camera, event,
cleaned_image,
telescope_id, picture,
boundary, min_neighbors)
quality_param_list.append(
[closest_approach, likelihood, num_diff_pixels])
index_best_parameters = optimize_param(quality_param_list, index)
index = index + 1
new_param = possible_cleaning_levels[index_best_parameters]
return new_param
# get a single image
for i, img in enumerate(source):
# Pick a (random) good looking image
if i > 147:
break
im = img.dl1.tel[0].image
print(img)
# Apply image cleaning
cleanmask = tailcuts_clean(
geom,
im,
picture_thresh=30,
boundary_thresh=5,
min_number_picture_neighbors=0,
)
if sum(cleanmask) == 0:
pass
else:
# Calculate image parameters
hillas = hillas_parameters(geom[cleanmask], im[cleanmask])
# Show the camera image and overlay Hillas ellipse and clean pixels
disp.image = im
disp.highlight_pixels(cleanmask, color="crimson")
disp.overlay_moments(hillas, color="red", linewidth=2)
plt.savefig("build/hillas_example.png", dpi=200)
def start(self):
disp = None
for event in tqdm(self.source,
desc='Tel{}'.format(self.tel),
total=self.reader.max_events,
disable=~self.progress):
self.log.debug(event.trig)
self.log.debug("Energy: {}".format(event.mc.energy))
self.calibrator.calibrate(event)
if disp is None:
x, y = event.inst.pixel_pos[self.tel]
focal_len = event.inst.optical_foclen[self.tel]
geom = CameraGeometry.guess(x, y, focal_len)
self.log.info(geom)
disp = CameraDisplay(geom)
# disp.enable_pixel_picker()
disp.add_colorbar()
if self.display:
plt.show(block=False)
# display the event
disp.axes.set_title('CT{:03d} ({}), event {:06d}'.format(
self.tel, geom.cam_id, event.r0.event_id)
)
if self.samples:
# display time-varying event
data = event.dl0.tel[self.tel].pe_samples[self.channel]
for ii in range(data.shape[1]):
disp.image = data[:, ii]
disp.set_limits_percent(70)
plt.suptitle("Sample {:03d}".format(ii))
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig('CT{:03d}_EV{:10d}_S{:02d}.png'
.format(self.tel, event.r0.event_id, ii))
else:
# display integrated event:
im = event.dl1.tel[self.tel].image[self.channel]
if self.clean:
mask = tailcuts_clean(geom, im, picture_thresh=10,
boundary_thresh=7)
im[~mask] = 0.0
disp.image = im
if self.hillas:
try:
ellipses = disp.axes.findobj(Ellipse)
if len(ellipses) > 0:
ellipses[0].remove()
params = hillas_parameters(pix_x=geom.pix_x,
pix_y=geom.pix_y, image=im)
disp.overlay_moments(params, color='pink', lw=3,
with_label=False)
except HillasParameterizationError:
pass
if self.display:
plt.pause(self.delay)
if self.write:
plt.savefig('CT{:03d}_EV{:010d}.png'
.format(self.tel, event.r0.event_id))
self.log.info("FINISHED READING DATA FILE")
if disp is None:
self.log.warning('No events for tel {} were found in {}. Try a '
'different EventIO file or another telescope'
.format(self.tel, self.infile),
)
pass
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
use_main_island=True,
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
superseeds the standard configuration
use_main_island: `bool` Use only the main island
to calculate DL1 parameters
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
cleaning_parameters = config["tailcut"]
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
if use_main_island:
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geometry[signal_pixels],
image[signal_pixels])
# Fill container
dl1_container.fill_hillas(hillas)
# convert ctapipe's width and length (in m) to deg:
foclen = subarray.tel[telescope_id].optics.equivalent_focal_length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.set_timing_features(camera_geometry[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geometry, image, signal_pixels)
dl1_container.set_concentration(camera_geometry, image, hillas)
dl1_container.n_pixels = n_pixels
dl1_container.n_islands = num_islands
dl1_container.set_telescope_info(subarray, telescope_id)
dl1_container.log_intensity = np.log10(dl1_container.intensity)
else:
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
return dl1_container
