def test_append_column_chunked_cta(cta_file, cta_config):
from aict_tools.io import read_telescope_data_chunked, read_data
from aict_tools.io import append_column_to_hdf5
new_column_name = 'foobar'
chunk_size = 125
table_name = 'telescope_events'
df = read_data(cta_file, table_name)
assert new_column_name not in df.columns
columns = cta_config.energy.columns_to_read_train
generator = read_telescope_data_chunked(cta_file, cta_config, chunk_size, columns=columns)
for df, start, stop in generator:
assert not df.empty
new_data = np.arange(start, stop, step=1)
append_column_to_hdf5(cta_file, new_data, table_name, new_column_name)
df = read_data(cta_file, table_name)
assert new_column_name in df.columns
assert np.array_equal(df.foobar, np.arange(0, len(df)))
df.set_index(
['run_id', 'array_event_id', 'telescope_id'],
drop=True,
verify_integrity=True,
inplace=True,
)
def test_append_column_chunked(hdf5_file):
from aict_tools.io import read_telescope_data_chunked, read_data
from aict_tools.io import append_column_to_hdf5
path, table_name, config = hdf5_file
new_column_name = 'foobar'
chunk_size = 125
df = read_data(path, table_name)
assert new_column_name not in df.columns
columns = config.energy.columns_to_read_train
generator = read_telescope_data_chunked(path, config, chunk_size, columns=columns)
for df, start, stop in generator:
assert not df.empty
new_data = np.arange(start, stop, step=1)
append_column_to_hdf5(path, new_data, table_name, new_column_name)
df = read_data(path, table_name)
assert new_column_name in df.columns
assert np.array_equal(df.foobar, np.arange(0, len(df)))
def test_read_data_cta(cta_file):
from aict_tools.io import read_data
df = read_data(cta_file, 'telescope_events')
assert 'telescope_id' in df.columns
df = read_data(cta_file, 'array_events')
assert 'array_event_id' in df.columns
def test_read_default_columns(hdf5_file):
from aict_tools.io import read_data, get_column_names_in_file
from pandas.util.testing import assert_frame_equal
path, table_name, config = hdf5_file
df = read_data(path, table_name)
cols = get_column_names_in_file(path, table_name)
df_all_columns = read_data(path, table_name, columns=cols)
assert_frame_equal(df, df_all_columns)
def test_append_column(hdf5_file):
from aict_tools.io import read_data
from aict_tools.io import append_column_to_hdf5
path, table_name, _ = hdf5_file
new_column_name = 'foobar'
df = read_data(path, table_name)
assert new_column_name not in df.columns
random_data = np.random.normal(size=len(df))
append_column_to_hdf5(path, random_data, table_name, new_column_name)
df = read_data(path, table_name)
assert new_column_name in df.columns
def test_read_data(hdf5_file):
from aict_tools.io import read_data
path, _, _ = hdf5_file
df = read_data(path, 'events')
assert 'run_id' in df.columns
assert 'width' in df.columns
def test_read_data(hdf5_file):
from aict_tools.io import read_data
path, _, _ = hdf5_file
df = read_data(path, "events")
assert "run_id" in df.columns
assert "width" in df.columns
def test_apply_regression_cta(temp_dir, cta_energy_model):
from aict_tools.scripts.apply_energy_regressor import main
from aict_tools.io import read_data
runner = CliRunner()
result = runner.invoke(
main,
[
"examples/cta_full_config.yaml",
os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"),
cta_energy_model,
"--yes",
],
)
if result.exit_code != 0:
print(result.output)
print_exception(*result.exc_info)
res_no_chunks = read_data(
os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"),
"/dl2/event/telescope/gamma_energy_prediction/tel_001",
)
result_2 = runner.invoke(
main,
[
"examples/cta_full_config.yaml",
os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"),
cta_energy_model,
"--yes",
"-N 50",
],
)
if result_2.exit_code != 0:
print(result_2.output)
print_exception(*result_2.exc_info)
res_chunks = read_data(
os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"),
"/dl2/event/telescope/gamma_energy_prediction/tel_001",
)
assert (res_no_chunks.all() == res_chunks.all()).all()
def test_horizontal_to_camera_cta():
from aict_tools.io import read_data
from aict_tools.cta_helpers import horizontal_to_camera
df = read_data("tests/cta_coord_test.hdf", "telescope_events")
expected_x = df.x
expected_y = df.y
transformed_x, transformed_y = horizontal_to_camera(
alt=90 - df.zd,
az=df.az,
alt_pointing=90 - df.zd_pointing,
az_pointing=df.az_pointing,
focal_length=df.focal_length,
)
assert_allclose(expected_x, transformed_x)
assert_allclose(expected_y, transformed_y)
def test_camera_to_horizontal():
from aict_tools.io import read_data
from aict_tools.cta_helpers import camera_to_horizontal
df = read_data("tests/cta_coord_test.hdf", "telescope_events")
expected_alt = df.alt
expected_az = df.az
transformed_alt, transformed_az = camera_to_horizontal(
x=df.x,
y=df.y,
alt_pointing=90 - df.zd_pointing,
az_pointing=df.az_pointing,
focal_length=df.focal_length,
)
assert_allclose(expected_alt, transformed_alt)
assert_allclose(expected_az, transformed_az)
from aict_tools.io import (read_data, append_column_to_hdf5)
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='PATH AND STUFF')
parser.add_argument('data_path', type=str)
parser.add_argument('column', type=str)
args = parser.parse_args()
a = read_data(args.data_path, 'array_events')
t = read_data(args.data_path,
'telescope_events')[['run_id', 'array_event_id', 'width']]
new_t = t.merge(a[['run_id', 'array_event_id', args.column]],
on=['run_id', 'array_event_id'],
how='left')
append_column_to_hdf5(args.data_path, new_t[args.column],
'telescope_events', args.column)
import argparse
from aict_tools.io import read_data
from cta_plots.spectrum import MCSpectrum
from aict_tools.cta_helpers import horizontal_to_camera_cta_simtel
from aict_tools.preprocessing import calc_true_disp
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('df_path', type=str)
parser.add_argument('output_path', type=str)
parser.add_argument('-correct_signs', action='store_true')
args = parser.parse_args()
df_path = args.df_path
output_path = args.output_path
events = read_data(df_path, 'array_events')
runs = read_data(df_path, 'runs')
mc_spectrum = MCSpectrum.from_cta_runs(runs)
#### implement cuts if needed
if args.correct_signs:
print('true sign cut')
true_x, true_y = horizontal_to_camera_cta_simtel(events)
true_disp, true_sign = calc_true_disp(true_x, true_y, events['x'],
events['y'],
np.deg2rad(df_tel['psi']))
predicted_sign_mask = (events['disp_prediction'] > 0)
true_sign_mask = (true_sign > 0)
correct_mask = (predicted_sign_mask == true_sign_mask)
events = events[correct_mask]
plot_multi_vs_energy)
from aict_tools.io import read_data
from astropy.coordinates.angle_utilities import angular_separation
from astropy import units as u
import argparse
import matplotlib.pyplot as plt
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='PATH AND STUFF')
parser.add_argument('df_path', type=str)
parser.add_argument('df_path_2', type=str)
parser.add_argument('output_folder', type=str)
args = parser.parse_args()
df = read_data(args.df_path, 'array_events')
df_2 = read_data(args.df_path_2, 'array_events')
recos = [
('source_alt_median', 'source_az_median', 'Median of telescope predictions', 'median'),
('source_alt_pairwise_mean_10.0', 'source_az_pairwise_mean_10.0', 'pairwise_mean averaging of telescope predictions', 'pairwise_mean_100'),
('source_alt_pairwise_median_10.0', 'source_az_pairwise_median_10.0', 'pairwise_median averaging of telescope predictions', 'pairwise_median_100'),
('source_alt_median_all', 'source_az_median_all', 'double median', 'median_all'),
('alt', 'az', 'HillasReconstructor', 'hillas')]
recos = [reco for reco in recos if reco[0] in df.columns]
for reco in recos:
## this removes all events hillas failed on!!!! these might still work with other methods!
df_ = df[['num_triggered_telescopes', 'mc_energy', 'mc_alt', 'mc_az', reco[0], reco[1]]].dropna(how='any')
theta = angular_separation(
return 0
return fig, ax
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='PATH AND STUFF')
parser.add_argument('df_path', type=str)
parser.add_argument('output_folder', type=str)
parser.add_argument('config_path', type=str)
args = parser.parse_args()
config = AICTConfig.from_yaml(args.config_path)
model_config = config.disp
# df = pd.read_hdf(args.df_path, 'array_events')
df = read_data(args.df_path, 'array_events')
recos = [
('source_alt_median', 'source_az_median',
'Median of telescope predictions', 'median'),
('source_alt_pairwise_mean_10.0', 'source_az_pairwise_mean_10.0',
'pairwise_mean averaging of telescope predictions',
'pairwise_mean_100'),
('source_alt_pairwise_median_10.0', 'source_az_pairwise_median_10.0',
'pairwise_median averaging of telescope predictions',
'pairwise_median_100'),
]
recos = [reco for reco in recos if reco[0] in df.columns]
plot_multi_vs_energy(df['num_triggered_telescopes'].values,
df['mc_energy'].values,
