def test_append_column_chunked_cta(cta_file, cta_config): from aict_tools.io import read_telescope_data_chunked, read_data from aict_tools.io import append_column_to_hdf5 new_column_name = 'foobar' chunk_size = 125 table_name = 'telescope_events' df = read_data(cta_file, table_name) assert new_column_name not in df.columns columns = cta_config.energy.columns_to_read_train generator = read_telescope_data_chunked(cta_file, cta_config, chunk_size, columns=columns) for df, start, stop in generator: assert not df.empty new_data = np.arange(start, stop, step=1) append_column_to_hdf5(cta_file, new_data, table_name, new_column_name) df = read_data(cta_file, table_name) assert new_column_name in df.columns assert np.array_equal(df.foobar, np.arange(0, len(df))) df.set_index( ['run_id', 'array_event_id', 'telescope_id'], drop=True, verify_integrity=True, inplace=True, )
def test_append_column_chunked(hdf5_file): from aict_tools.io import read_telescope_data_chunked, read_data from aict_tools.io import append_column_to_hdf5 path, table_name, config = hdf5_file new_column_name = 'foobar' chunk_size = 125 df = read_data(path, table_name) assert new_column_name not in df.columns columns = config.energy.columns_to_read_train generator = read_telescope_data_chunked(path, config, chunk_size, columns=columns) for df, start, stop in generator: assert not df.empty new_data = np.arange(start, stop, step=1) append_column_to_hdf5(path, new_data, table_name, new_column_name) df = read_data(path, table_name) assert new_column_name in df.columns assert np.array_equal(df.foobar, np.arange(0, len(df)))
def test_read_data_cta(cta_file): from aict_tools.io import read_data df = read_data(cta_file, 'telescope_events') assert 'telescope_id' in df.columns df = read_data(cta_file, 'array_events') assert 'array_event_id' in df.columns
def test_read_default_columns(hdf5_file): from aict_tools.io import read_data, get_column_names_in_file from pandas.util.testing import assert_frame_equal path, table_name, config = hdf5_file df = read_data(path, table_name) cols = get_column_names_in_file(path, table_name) df_all_columns = read_data(path, table_name, columns=cols) assert_frame_equal(df, df_all_columns)
def test_append_column(hdf5_file): from aict_tools.io import read_data from aict_tools.io import append_column_to_hdf5 path, table_name, _ = hdf5_file new_column_name = 'foobar' df = read_data(path, table_name) assert new_column_name not in df.columns random_data = np.random.normal(size=len(df)) append_column_to_hdf5(path, random_data, table_name, new_column_name) df = read_data(path, table_name) assert new_column_name in df.columns
def test_read_data(hdf5_file): from aict_tools.io import read_data path, _, _ = hdf5_file df = read_data(path, 'events') assert 'run_id' in df.columns assert 'width' in df.columns
def test_read_data(hdf5_file): from aict_tools.io import read_data path, _, _ = hdf5_file df = read_data(path, "events") assert "run_id" in df.columns assert "width" in df.columns
def test_apply_regression_cta(temp_dir, cta_energy_model): from aict_tools.scripts.apply_energy_regressor import main from aict_tools.io import read_data runner = CliRunner() result = runner.invoke( main, [ "examples/cta_full_config.yaml", os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"), cta_energy_model, "--yes", ], ) if result.exit_code != 0: print(result.output) print_exception(*result.exc_info) res_no_chunks = read_data( os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"), "/dl2/event/telescope/gamma_energy_prediction/tel_001", ) result_2 = runner.invoke( main, [ "examples/cta_full_config.yaml", os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"), cta_energy_model, "--yes", "-N 50", ], ) if result_2.exit_code != 0: print(result_2.output) print_exception(*result_2.exc_info) res_chunks = read_data( os.path.join(temp_dir, "cta_gammas_diffuse.dl1.h5"), "/dl2/event/telescope/gamma_energy_prediction/tel_001", ) assert (res_no_chunks.all() == res_chunks.all()).all()
def test_horizontal_to_camera_cta(): from aict_tools.io import read_data from aict_tools.cta_helpers import horizontal_to_camera df = read_data("tests/cta_coord_test.hdf", "telescope_events") expected_x = df.x expected_y = df.y transformed_x, transformed_y = horizontal_to_camera( alt=90 - df.zd, az=df.az, alt_pointing=90 - df.zd_pointing, az_pointing=df.az_pointing, focal_length=df.focal_length, ) assert_allclose(expected_x, transformed_x) assert_allclose(expected_y, transformed_y)
def test_camera_to_horizontal(): from aict_tools.io import read_data from aict_tools.cta_helpers import camera_to_horizontal df = read_data("tests/cta_coord_test.hdf", "telescope_events") expected_alt = df.alt expected_az = df.az transformed_alt, transformed_az = camera_to_horizontal( x=df.x, y=df.y, alt_pointing=90 - df.zd_pointing, az_pointing=df.az_pointing, focal_length=df.focal_length, ) assert_allclose(expected_alt, transformed_alt) assert_allclose(expected_az, transformed_az)
from aict_tools.io import (read_data, append_column_to_hdf5) import argparse if __name__ == '__main__': parser = argparse.ArgumentParser(description='PATH AND STUFF') parser.add_argument('data_path', type=str) parser.add_argument('column', type=str) args = parser.parse_args() a = read_data(args.data_path, 'array_events') t = read_data(args.data_path, 'telescope_events')[['run_id', 'array_event_id', 'width']] new_t = t.merge(a[['run_id', 'array_event_id', args.column]], on=['run_id', 'array_event_id'], how='left') append_column_to_hdf5(args.data_path, new_t[args.column], 'telescope_events', args.column)
import argparse from aict_tools.io import read_data from cta_plots.spectrum import MCSpectrum from aict_tools.cta_helpers import horizontal_to_camera_cta_simtel from aict_tools.preprocessing import calc_true_disp if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('df_path', type=str) parser.add_argument('output_path', type=str) parser.add_argument('-correct_signs', action='store_true') args = parser.parse_args() df_path = args.df_path output_path = args.output_path events = read_data(df_path, 'array_events') runs = read_data(df_path, 'runs') mc_spectrum = MCSpectrum.from_cta_runs(runs) #### implement cuts if needed if args.correct_signs: print('true sign cut') true_x, true_y = horizontal_to_camera_cta_simtel(events) true_disp, true_sign = calc_true_disp(true_x, true_y, events['x'], events['y'], np.deg2rad(df_tel['psi'])) predicted_sign_mask = (events['disp_prediction'] > 0) true_sign_mask = (true_sign > 0) correct_mask = (predicted_sign_mask == true_sign_mask) events = events[correct_mask]
plot_multi_vs_energy) from aict_tools.io import read_data from astropy.coordinates.angle_utilities import angular_separation from astropy import units as u import argparse import matplotlib.pyplot as plt if __name__ == '__main__': parser = argparse.ArgumentParser(description='PATH AND STUFF') parser.add_argument('df_path', type=str) parser.add_argument('df_path_2', type=str) parser.add_argument('output_folder', type=str) args = parser.parse_args() df = read_data(args.df_path, 'array_events') df_2 = read_data(args.df_path_2, 'array_events') recos = [ ('source_alt_median', 'source_az_median', 'Median of telescope predictions', 'median'), ('source_alt_pairwise_mean_10.0', 'source_az_pairwise_mean_10.0', 'pairwise_mean averaging of telescope predictions', 'pairwise_mean_100'), ('source_alt_pairwise_median_10.0', 'source_az_pairwise_median_10.0', 'pairwise_median averaging of telescope predictions', 'pairwise_median_100'), ('source_alt_median_all', 'source_az_median_all', 'double median', 'median_all'), ('alt', 'az', 'HillasReconstructor', 'hillas')] recos = [reco for reco in recos if reco[0] in df.columns] for reco in recos: ## this removes all events hillas failed on!!!! these might still work with other methods! df_ = df[['num_triggered_telescopes', 'mc_energy', 'mc_alt', 'mc_az', reco[0], reco[1]]].dropna(how='any') theta = angular_separation(
return 0 return fig, ax if __name__ == '__main__': parser = argparse.ArgumentParser(description='PATH AND STUFF') parser.add_argument('df_path', type=str) parser.add_argument('output_folder', type=str) parser.add_argument('config_path', type=str) args = parser.parse_args() config = AICTConfig.from_yaml(args.config_path) model_config = config.disp # df = pd.read_hdf(args.df_path, 'array_events') df = read_data(args.df_path, 'array_events') recos = [ ('source_alt_median', 'source_az_median', 'Median of telescope predictions', 'median'), ('source_alt_pairwise_mean_10.0', 'source_az_pairwise_mean_10.0', 'pairwise_mean averaging of telescope predictions', 'pairwise_mean_100'), ('source_alt_pairwise_median_10.0', 'source_az_pairwise_median_10.0', 'pairwise_median averaging of telescope predictions', 'pairwise_median_100'), ] recos = [reco for reco in recos if reco[0] in df.columns] plot_multi_vs_energy(df['num_triggered_telescopes'].values, df['mc_energy'].values,