def main():
if args.run_number:
file_list = sorted([
os.path.join(args.srcdir, f) for f in os.listdir(args.srcdir)
if (f.endswith('.h5') and args.run_number in f)
])
else:
file_list = sorted([
os.path.join(args.srcdir, f) for f in os.listdir(args.srcdir)
if f.endswith('.h5')
])
if args.noimage:
keys = get_dataset_keys(file_list[0])
for k in keys:
if 'image' in k:
keys.remove(k)
else:
keys = None
if args.smart:
smart_merge_h5files(file_list, args.outfile, node_keys=keys)
else:
auto_merge_h5files(file_list, args.outfile, nodes_keys=keys)
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
geom = CameraGeometry.from_name('LSTCam-002')
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend(['wl', 'r', 'leakage', 'n_islands', 'intercept', 'time_gradient'])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii%10000 == 0:
print(ii)
image = row['image']
pulse_time = row['pulse_time']
signal_pixels = tailcuts_clean(geom, image, **config['tailcut'])
if image[signal_pixels].shape[0] > 0:
num_islands, island_labels = number_of_islands(geom, signal_pixels)
hillas = hillas_parameters(geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(geom[signal_pixels],
image[signal_pixels],
pulse_time[signal_pixels],
hillas)
dl1_container.set_leakage(geom, image, signal_pixels)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width.value
dl1_container.length = length.value
dl1_container.r = np.sqrt(dl1_container.x**2 + dl1_container.y**2)
for p in parameters_to_update:
params[ii][p] = Quantity(dl1_container[p]).value
else:
for p in parameters_to_update:
params[ii][p] = 0
output.root[dl1_params_lstcam_key][:] = params
def main():
file_list = [
args.srcdir + '/' + f for f in os.listdir(args.srcdir)
if f.endswith('.h5')
]
if args.noimage:
keys = get_dataset_keys(file_list[0])
for k in keys:
if 'image' in k:
keys.remove(k)
else:
keys = None
if args.smart:
smart_merge_h5files(file_list, args.outfile, node_keys=keys)
else:
auto_merge_h5files(file_list, args.outfile, nodes_keys=keys)
def main():
if args.run_number:
run = f'Run{args.run_number:05d}'
file_list = sorted(
filter(lambda f: run in f,
glob(os.path.join(args.srcdir, args.pattern))))
else:
file_list = sorted(glob(os.path.join(args.srcdir, args.pattern)))
if args.noimage:
keys = get_dataset_keys(file_list[0])
keys = {k for k in keys if 'image' not in k}
else:
keys = None
if args.smart:
smart_merge_h5files(file_list, args.outfile, node_keys=keys)
else:
auto_merge_h5files(file_list, args.outfile, nodes_keys=keys)
def main():
args = parser.parse_args()
if args.run_number:
run = f'Run{args.run_number:05d}'
file_list = sorted(filter(
lambda f: run in f,
glob(os.path.join(args.input_dir, args.pattern))
))
else:
file_list = sorted(glob(os.path.join(args.input_dir, args.pattern)))
if args.no_image:
keys = get_dataset_keys(file_list[0])
keys = {k for k in keys if 'image' not in k}
else:
keys = None
auto_merge_h5files(
file_list,
args.output_file,
nodes_keys=keys,
progress_bar=not args.no_progress
)
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config, read_configuration_file(args.config_file))
else:
config = std_config
log.info(f"Tailcut config used: {config['tailcut']}")
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file, path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'log_intensity'
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file], args.output_file, nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, **config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(island_labels.astype(np.int))
n_pixels_on_island[0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels], image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(camera_geom[signal_pixels],
image[signal_pixels],
peak_time[signal_pixels],
hillas)
dl1_container.set_leakage(camera_geom, image, signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m),
params['src_x'][ii],
params['src_y'][ii]
)
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def main():
args = parser.parse_args()
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(
os.path.abspath(args.config_file))
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
data = pd.read_hdf(args.input_file, key=dl1_params_lstcam_key)
if 'lh_fit_config' in config.keys():
lhfit_data = pd.read_hdf(args.input_file,
key=dl1_likelihood_params_lstcam_key)
if np.all(lhfit_data['obs_id'] == data['obs_id']) & np.all(
lhfit_data['event_id'] == data['event_id']):
lhfit_data.drop({'obs_id', 'event_id'}, axis=1, inplace=True)
lhfit_keys = lhfit_data.keys()
data = pd.concat([data, lhfit_data], axis=1)
# if real data, add deltat t to dataframe keys
data = add_delta_t_key(data)
# Dealing with pointing missing values. This happened when `ucts_time` was invalid.
if 'alt_tel' in data.columns and 'az_tel' in data.columns \
and (np.isnan(data.alt_tel).any() or np.isnan(data.az_tel).any()):
# make sure there is a least one good pointing value to interp from.
if np.isfinite(data.alt_tel).any() and np.isfinite(data.az_tel).any():
data = impute_pointing(data)
else:
data.alt_tel = -np.pi / 2.
data.az_tel = -np.pi / 2.
# Get trained RF path for reconstruction:
file_reg_energy = os.path.join(args.path_models, 'reg_energy.sav')
file_cls_gh = os.path.join(args.path_models, 'cls_gh.sav')
if config['disp_method'] == 'disp_vector':
file_disp_vector = os.path.join(args.path_models,
'reg_disp_vector.sav')
elif config['disp_method'] == 'disp_norm_sign':
file_disp_norm = os.path.join(args.path_models, 'reg_disp_norm.sav')
file_disp_sign = os.path.join(args.path_models, 'cls_disp_sign.sav')
subarray_info = SubarrayDescription.from_hdf(args.input_file)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length
# Apply the models to the data
# Source-independent analysis
if not config['source_dependent']:
data = filter_events(
data,
filters=config["events_filters"],
finite_params=config['energy_regression_features'] +
config['disp_regression_features'] +
config['particle_classification_features'] +
config['disp_classification_features'],
)
if config['disp_method'] == 'disp_vector':
dl2 = dl1_to_dl2.apply_models(data,
file_cls_gh,
file_reg_energy,
reg_disp_vector=file_disp_vector,
focal_length=focal_length,
custom_config=config)
elif config['disp_method'] == 'disp_norm_sign':
dl2 = dl1_to_dl2.apply_models(data,
file_cls_gh,
file_reg_energy,
reg_disp_norm=file_disp_norm,
cls_disp_sign=file_disp_sign,
focal_length=focal_length,
custom_config=config)
# Source-dependent analysis
if config['source_dependent']:
# if source-dependent parameters are already in dl1 data, just read those data.
if dl1_params_src_dep_lstcam_key in get_dataset_keys(args.input_file):
data_srcdep = get_srcdep_params(args.input_file)
# if not, source-dependent parameters are added now
else:
data_srcdep = pd.concat(dl1_to_dl2.get_source_dependent_parameters(
data, config, focal_length=focal_length),
axis=1)
dl2_srcdep_dict = {}
srcindep_keys = data.keys()
srcdep_assumed_positions = data_srcdep.columns.levels[0]
for i, k in enumerate(srcdep_assumed_positions):
data_with_srcdep_param = pd.concat([data, data_srcdep[k]], axis=1)
data_with_srcdep_param = filter_events(
data_with_srcdep_param,
filters=config["events_filters"],
finite_params=config['energy_regression_features'] +
config['disp_regression_features'] +
config['particle_classification_features'] +
config['disp_classification_features'],
)
if config['disp_method'] == 'disp_vector':
dl2_df = dl1_to_dl2.apply_models(
data_with_srcdep_param,
file_cls_gh,
file_reg_energy,
reg_disp_vector=file_disp_vector,
focal_length=focal_length,
custom_config=config)
elif config['disp_method'] == 'disp_norm_sign':
dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param,
file_cls_gh,
file_reg_energy,
reg_disp_norm=file_disp_norm,
cls_disp_sign=file_disp_sign,
focal_length=focal_length,
custom_config=config)
dl2_srcdep = dl2_df.drop(srcindep_keys, axis=1)
dl2_srcdep_dict[k] = dl2_srcdep
if i == 0:
dl2_srcindep = dl2_df[srcindep_keys]
os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(
args.output_dir,
os.path.basename(args.input_file).replace('dl1', 'dl2', 1))
if os.path.exists(output_file):
raise IOError(output_file + ' exists, exiting.')
dl1_keys = get_dataset_keys(args.input_file)
if dl1_images_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_images_lstcam_key)
if dl1_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_lstcam_key)
if dl1_params_src_dep_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_src_dep_lstcam_key)
if dl1_likelihood_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_likelihood_params_lstcam_key)
metadata = global_metadata()
write_metadata(metadata, output_file)
with open_file(args.input_file, 'r') as h5in:
with open_file(output_file, 'a') as h5out:
# Write the selected DL1 info
for k in dl1_keys:
if not k.startswith('/'):
k = '/' + k
path, name = k.rsplit('/', 1)
if path not in h5out:
grouppath, groupname = path.rsplit('/', 1)
g = h5out.create_group(grouppath,
groupname,
createparents=True)
else:
g = h5out.get_node(path)
h5in.copy_node(k, g, overwrite=True)
# need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata
if not config['source_dependent']:
if 'lh_fit_config' not in config.keys():
write_dl2_dataframe(dl2, output_file, config=config, meta=metadata)
else:
dl2_onlylhfit = dl2[lhfit_keys]
dl2.drop(lhfit_keys, axis=1, inplace=True)
write_dl2_dataframe(dl2, output_file, config=config, meta=metadata)
write_dataframe(dl2_onlylhfit,
output_file,
dl2_likelihood_params_lstcam_key,
config=config,
meta=metadata)
else:
write_dl2_dataframe(dl2_srcindep,
output_file,
config=config,
meta=metadata)
write_dataframe(pd.concat(dl2_srcdep_dict, axis=1),
output_file,
dl2_params_src_dep_lstcam_key,
config=config,
meta=metadata)
def main():
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
print(config['tailcut'])
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = list(HillasParametersContainer().keys())
parameters_to_update.extend([
'concentration_cog',
'concentration_core',
'concentration_pixel',
'leakage_intensity_width_1',
'leakage_intensity_width_2',
'leakage_pixels_width_1',
'leakage_pixels_width_2',
'n_islands',
'intercept',
'time_gradient',
'n_pixels',
'wl',
'r',
])
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
if ii % 10000 == 0:
print(ii)
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image,
**config['tailcut'])
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.r = np.sqrt(dl1_container.x**2 +
dl1_container.y**2)
else:
# for consistency with r0_to_dl1.py:
for key in dl1_container.keys():
dl1_container[key] = \
u.Quantity(0, dl1_container.fields[key].unit)
dl1_container.width = u.Quantity(np.nan, u.m)
dl1_container.length = u.Quantity(np.nan, u.m)
dl1_container.wl = u.Quantity(np.nan, u.m)
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
def main():
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(
os.path.abspath(args.config_file))
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
data = pd.read_hdf(args.input_file, key=dl1_params_lstcam_key)
if config['source_dependent']:
data_src_dep = pd.read_hdf(args.input_file,
key=dl1_params_src_dep_lstcam_key)
data = pd.concat([data, data_src_dep], axis=1)
# Dealing with pointing missing values. This happened when `ucts_time` was invalid.
if 'alt_tel' in data.columns and 'az_tel' in data.columns \
and (np.isnan(data.alt_tel).any() or np.isnan(data.az_tel).any()):
# make sure there is a least one good pointing value to interp from.
if np.isfinite(data.alt_tel).any() and np.isfinite(data.az_tel).any():
data = impute_pointing(data)
else:
data.alt_tel = -np.pi / 2.
data.az_tel = -np.pi / 2.
data = filter_events(
data,
filters=config["events_filters"],
finite_params=config['regression_features'] +
config['classification_features'],
)
#Load the trained RF for reconstruction:
fileE = args.path_models + "/reg_energy.sav"
fileD = args.path_models + "/reg_disp_vector.sav"
fileH = args.path_models + "/cls_gh.sav"
reg_energy = joblib.load(fileE)
reg_disp_vector = joblib.load(fileD)
cls_gh = joblib.load(fileH)
#Apply the models to the data
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_vector,
custom_config=config)
os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(
args.output_dir,
os.path.basename(args.input_file).replace('dl1', 'dl2'))
if os.path.exists(output_file):
raise IOError(output_file + ' exists, exiting.')
dl1_keys = get_dataset_keys(args.input_file)
if dl1_images_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_images_lstcam_key)
if dl1_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_lstcam_key)
if dl1_params_src_dep_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_src_dep_lstcam_key)
with open_file(args.input_file, 'r') as h5in:
with open_file(output_file, 'a') as h5out:
# Write the selected DL1 info
for k in dl1_keys:
if not k.startswith('/'):
k = '/' + k
path, name = k.rsplit('/', 1)
if path not in h5out:
grouppath, groupname = path.rsplit('/', 1)
g = h5out.create_group(grouppath,
groupname,
createparents=True)
else:
g = h5out.get_node(path)
h5in.copy_node(k, g, overwrite=True)
write_dl2_dataframe(dl2, output_file)
def main():
args = parser.parse_args()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if Path(args.output_file).exists():
log.critical(f'Output file {args.output_file} already exists')
sys.exit(1)
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
with tables.open_file(args.input_file, 'r') as f:
is_simulation = 'simulation' in f.root
increase_nsb = False
increase_psf = False
if "image_modifier" in config:
imconfig = config["image_modifier"]
increase_nsb = imconfig["increase_nsb"]
increase_psf = imconfig["increase_psf"]
if increase_nsb or increase_psf:
log.info(f"image_modifier configuration: {imconfig}")
extra_noise_in_dim_pixels = imconfig["extra_noise_in_dim_pixels"]
extra_bias_in_dim_pixels = imconfig["extra_bias_in_dim_pixels"]
transition_charge = imconfig["transition_charge"]
extra_noise_in_bright_pixels = imconfig["extra_noise_in_bright_pixels"]
smeared_light_fraction = imconfig["smeared_light_fraction"]
if (increase_nsb or increase_psf):
log.info(
"NOTE: Using the image_modifier options means images will "
"not be saved.")
args.no_image = True
if is_simulation:
args.pedestal_cleaning = False
if args.pedestal_cleaning:
log.info("Pedestal cleaning")
clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
sigma = config[clean_method_name]['sigma']
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(
f"Fraction of pixel cleaning thresholds above picture thr.:"
f"{np.sum(pedestal_thresh > pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")
else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Tailcut config used: {config['tailcut']}")
use_dynamic_cleaning = False
if 'apply' in config['dynamic_cleaning']:
use_dynamic_cleaning = config['dynamic_cleaning']['apply']
if use_dynamic_cleaning:
THRESHOLD_DYNAMIC_CLEANING = config['dynamic_cleaning']['threshold']
FRACTION_CLEANING_SIZE = config['dynamic_cleaning'][
'fraction_cleaning_intensity']
log.info(
"Using dynamic cleaning for events with average size of the "
f"3 most brighest pixels > {config['dynamic_cleaning']['threshold']} p.e"
)
log.info(
"Remove from image pixels which have charge below "
f"= {config['dynamic_cleaning']['fraction_cleaning_intensity']} * average size"
)
use_only_main_island = True
if "use_only_main_island" in config[clean_method_name]:
use_only_main_island = config[clean_method_name][
"use_only_main_island"]
delta_time = None
if "delta_time" in config[clean_method_name]:
delta_time = config[clean_method_name]["delta_time"]
subarray_info = SubarrayDescription.from_hdf(args.input_file)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
optics = subarray_info.tel[tel_id].optics
camera_geom = subarray_info.tel[tel_id].camera.geometry
dl1_container = DL1ParametersContainer()
parameters_to_update = [
'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
'concentration_pixel', 'leakage_intensity_width_1',
'leakage_intensity_width_2', 'leakage_pixels_width_1',
'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
'n_pixels', 'wl', 'log_intensity'
]
nodes_keys = get_dataset_keys(args.input_file)
if args.no_image:
nodes_keys.remove(dl1_images_lstcam_key)
metadata = global_metadata()
with tables.open_file(args.input_file, mode='r') as infile:
image_table = infile.root[dl1_images_lstcam_key]
dl1_params_input = infile.root[dl1_params_lstcam_key].colnames
disp_params = {
'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
uncertainty_params = {'width_uncertainty', 'length_uncertainty'}
if set(dl1_params_input).intersection(uncertainty_params):
parameters_to_update.extend(uncertainty_params)
if increase_nsb:
rng = np.random.default_rng(
infile.root.dl1.event.subarray.trigger.col('obs_id')[0])
if increase_psf:
set_numba_seed(
infile.root.dl1.event.subarray.trigger.col('obs_id')[0])
image_mask_save = not args.no_image and 'image_mask' in infile.root[
dl1_images_lstcam_key].colnames
with tables.open_file(args.output_file,
mode='a',
filters=HDF5_ZSTD_FILTERS) as outfile:
copy_h5_nodes(infile, outfile, nodes=nodes_keys)
add_source_filenames(outfile, [args.input_file])
params = outfile.root[dl1_params_lstcam_key].read()
if image_mask_save:
image_mask = outfile.root[dl1_images_lstcam_key].col(
'image_mask')
# need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata
for k, item in metadata.as_dict().items():
outfile.root[dl1_params_lstcam_key].attrs[k] = item
outfile.root[dl1_params_lstcam_key].attrs["config"] = str(config)
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
if increase_nsb:
# Add noise in pixels, to adjust MC to data noise levels.
# TO BE DONE: in case of "pedestal cleaning" (not used now
# in MC) we should recalculate picture_th above!
image = add_noise_in_pixels(rng, image,
extra_noise_in_dim_pixels,
extra_bias_in_dim_pixels,
transition_charge,
extra_noise_in_bright_pixels)
if increase_psf:
image = random_psf_smearer(
image, smeared_light_fraction,
camera_geom.neighbor_matrix_sparse.indices,
camera_geom.neighbor_matrix_sparse.indptr)
signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
boundary_th, isolated_pixels,
min_n_neighbors)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# if delta_time has been set, we require at least one
# neighbor within delta_time to accept a pixel in the image:
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(
camera_geom, signal_pixels, cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
if use_dynamic_cleaning:
new_mask = apply_dynamic_cleaning(
image, signal_pixels, THRESHOLD_DYNAMIC_CLEANING,
FRACTION_CLEANING_SIZE)
signal_pixels = new_mask
# count a number of islands after all of the image cleaning steps
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
dl1_container.n_islands = num_islands
n_pixels_on_island = np.bincount(
island_labels.astype(np.int64))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
if use_only_main_island:
signal_pixels[
island_labels != max_island_label] = False
# count the surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
dl1_container.n_pixels = n_pixels
if n_pixels > 0:
parametrize_image(
image=image,
peak_time=peak_time,
signal_pixels=signal_pixels,
camera_geometry=camera_geom,
focal_length=optics.equivalent_focal_length,
dl1_container=dl1_container,
)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m), params['src_x'][ii],
params['src_y'][ii])
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
if image_mask_save:
image_mask[ii] = signal_pixels
outfile.root[dl1_params_lstcam_key][:] = params
if image_mask_save:
outfile.root[dl1_images_lstcam_key].modify_column(
colname='image_mask', column=image_mask)
write_metadata(metadata, args.output_file)
#Apply the models to the data
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_vector,
custom_config=config)
if args.storeresults == True:
#Store results
os.makedirs(args.outdir, exist_ok=True)
outfile = args.outdir + '/dl2_' + os.path.basename(
args.datafile).split('.')[0] + '.h5'
dl2.to_hdf(outfile, key="events/LSTCam", mode="w")
keys = get_dataset_keys(dl1_file)
groups = set([k.split('/')[0] for k in keys])
groups.remove('events') # we don't want to copy DL1 events
f1 = tables.open_file(dl1_file)
with tables.open_file(outfile, mode='a') as dl2_file:
nodes = {}
for g in groups:
nodes[g] = f1.copy_node('/',
name=g,
newparent=dl2_file.root,
newname=g,
recursive=True)
def main():
std_config = get_standard_config()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
if args.pedestal_cleaning:
print("Pedestal cleaning")
clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
sigma = config[clean_method_name]['sigma']
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(
f"Fraction of pixel cleaning thresholds above picture thr.:"
f"{np.sum(pedestal_thresh>pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")
else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Tailcut config used: {config['tailcut']}")
use_only_main_island = True
if "use_only_main_island" in config[clean_method_name]:
use_only_main_island = config[clean_method_name][
"use_only_main_island"]
delta_time = None
if "delta_time" in config[clean_method_name]:
delta_time = config[clean_method_name]["delta_time"]
foclen = OpticsDescription.from_name('LST').equivalent_focal_length
cam_table = Table.read(args.input_file,
path="instrument/telescope/camera/LSTCam")
camera_geom = CameraGeometry.from_table(cam_table)
dl1_container = DL1ParametersContainer()
parameters_to_update = [
'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
'concentration_pixel', 'leakage_intensity_width_1',
'leakage_intensity_width_2', 'leakage_pixels_width_1',
'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
'n_pixels', 'wl', 'log_intensity'
]
nodes_keys = get_dataset_keys(args.input_file)
if args.noimage:
nodes_keys.remove(dl1_images_lstcam_key)
auto_merge_h5files([args.input_file],
args.output_file,
nodes_keys=nodes_keys)
with tables.open_file(args.input_file, mode='r') as input:
image_table = input.root[dl1_images_lstcam_key]
dl1_params_input = input.root[dl1_params_lstcam_key].colnames
disp_params = {
'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
with tables.open_file(args.output_file, mode='a') as output:
params = output.root[dl1_params_lstcam_key].read()
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
boundary_th, isolated_pixels,
min_n_neighbors)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
n_pixels_on_island = np.bincount(
island_labels.astype(np.int64))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
if use_only_main_island:
signal_pixels[
island_labels != max_island_label] = False
# if delta_time has been set, we require at least one
# neighbor within delta_time to accept a pixel in the image:
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(
camera_geom, signal_pixels, cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
# count the surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
hillas = hillas_parameters(camera_geom[signal_pixels],
image[signal_pixels])
dl1_container.fill_hillas(hillas)
dl1_container.set_timing_features(
camera_geom[signal_pixels], image[signal_pixels],
peak_time[signal_pixels], hillas)
dl1_container.set_leakage(camera_geom, image,
signal_pixels)
dl1_container.set_concentration(
camera_geom, image, hillas)
dl1_container.n_islands = num_islands
dl1_container.wl = dl1_container.width / dl1_container.length
dl1_container.n_pixels = n_pixels
width = np.rad2deg(
np.arctan2(dl1_container.width, foclen))
length = np.rad2deg(
np.arctan2(dl1_container.length, foclen))
dl1_container.width = width
dl1_container.length = length
dl1_container.log_intensity = np.log10(
dl1_container.intensity)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m), params['src_x'][ii],
params['src_y'][ii])
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
output.root[dl1_params_lstcam_key][:] = params
'-o',
action='store',
type=str,
dest='outfile',
help='Path of the resulting merged file',
default='merge.h5')
args = parser.parse_args()
if __name__ == '__main__':
file_list = [
args.srcdir + '/' + f for f in os.listdir(args.srcdir)
if f.endswith('.h5')
]
keys = get_dataset_keys(file_list[0])
groups = set([k.split('/')[0] for k in keys])
f1 = tables.open_file(file_list[0])
merge_file = tables.open_file(args.outfile, 'w')
nodes = {}
for g in groups:
nodes[g] = f1.copy_node('/',
name=g,
newparent=merge_file.root,
newname=g,
recursive=True)
for filename in file_list[1:]:
with tables.open_file(filename) as file:
def main():
custom_config = {}
if args.config_file is not None:
try:
custom_config = read_configuration_file(
os.path.abspath(args.config_file))
except ("Custom configuration could not be loaded !!!"):
pass
config = replace_config(standard_config, custom_config)
data = pd.read_hdf(args.input_file, key=dl1_params_lstcam_key)
# if real data, add deltat t to dataframe keys
data = add_delta_t_key(data)
# Dealing with pointing missing values. This happened when `ucts_time` was invalid.
if 'alt_tel' in data.columns and 'az_tel' in data.columns \
and (np.isnan(data.alt_tel).any() or np.isnan(data.az_tel).any()):
# make sure there is a least one good pointing value to interp from.
if np.isfinite(data.alt_tel).any() and np.isfinite(data.az_tel).any():
data = impute_pointing(data)
else:
data.alt_tel = -np.pi / 2.
data.az_tel = -np.pi / 2.
# Load the trained RF for reconstruction:
fileE = args.path_models + "/reg_energy.sav"
fileD = args.path_models + "/reg_disp_vector.sav"
fileH = args.path_models + "/cls_gh.sav"
reg_energy = joblib.load(fileE)
reg_disp_vector = joblib.load(fileD)
cls_gh = joblib.load(fileH)
subarray_info = SubarrayDescription.from_hdf(args.input_file)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
focal_length = subarray_info.tel[tel_id].optics.equivalent_focal_length
# Apply the models to the data
# Source-independent analysis
if not config['source_dependent']:
data = filter_events(
data,
filters=config["events_filters"],
finite_params=config['regression_features'] +
config['classification_features'],
)
dl2 = dl1_to_dl2.apply_models(data,
cls_gh,
reg_energy,
reg_disp_vector,
focal_length=focal_length,
custom_config=config)
# Source-dependent analysis
if config['source_dependent']:
data_srcdep = pd.read_hdf(args.input_file,
key=dl1_params_src_dep_lstcam_key)
data_srcdep.columns = pd.MultiIndex.from_tuples([
tuple(col[1:-1].replace('\'', '').replace(' ', '').split(","))
for col in data_srcdep.columns
])
dl2_srcdep_dict = {}
for i, k in enumerate(data_srcdep.columns.levels[0]):
data_with_srcdep_param = pd.concat([data, data_srcdep[k]], axis=1)
data_with_srcdep_param = filter_events(
data_with_srcdep_param,
filters=config["events_filters"],
finite_params=config['regression_features'] +
config['classification_features'],
)
dl2_df = dl1_to_dl2.apply_models(data_with_srcdep_param,
cls_gh,
reg_energy,
reg_disp_vector,
focal_length=focal_length,
custom_config=config)
dl2_srcdep = dl2_df.drop(data.keys(), axis=1)
dl2_srcdep_dict[k] = dl2_srcdep
if i == 0:
dl2_srcindep = dl2_df.drop(data_srcdep[k].keys(), axis=1)
os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(
args.output_dir,
os.path.basename(args.input_file).replace('dl1', 'dl2'))
if os.path.exists(output_file):
raise IOError(output_file + ' exists, exiting.')
dl1_keys = get_dataset_keys(args.input_file)
if dl1_images_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_images_lstcam_key)
if dl1_params_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_lstcam_key)
if dl1_params_src_dep_lstcam_key in dl1_keys:
dl1_keys.remove(dl1_params_src_dep_lstcam_key)
with open_file(args.input_file, 'r') as h5in:
with open_file(output_file, 'a') as h5out:
# Write the selected DL1 info
for k in dl1_keys:
if not k.startswith('/'):
k = '/' + k
path, name = k.rsplit('/', 1)
if path not in h5out:
grouppath, groupname = path.rsplit('/', 1)
g = h5out.create_group(grouppath,
groupname,
createparents=True)
else:
g = h5out.get_node(path)
h5in.copy_node(k, g, overwrite=True)
if not config['source_dependent']:
write_dl2_dataframe(dl2, output_file)
else:
write_dl2_dataframe(dl2_srcindep, output_file)
write_dataframe(pd.concat(dl2_srcdep_dict, axis=1), output_file,
dl2_params_src_dep_lstcam_key)
