def test_template_read():
# Now lets check out our reading Chain
# Create our fitter object
fitter = TemplateFitter()
# Get our example data file (10 events of 1 TeV at 0 Alt, 0 Az)
data_dir = pkg_resources.resource_filename('template_builder', 'data/')
# Which needs to actually be there
assert data_dir is not None
data_dir += "gamma_HESS_example.simhess.gz"
# Read in the file
amp, raw_x, raw_y = fitter.read_templates(data_dir)
# First check our output is empty
assert amp.keys() is not None
# Check our template parameters are correct
assert list(amp.keys())[0][0] == 0. # Alt
assert list(amp.keys())[0][1] == 0. # Az
assert list(amp.keys())[0][2] == 1. # Energy
# Can't be sure of the exact template content, but at least check the values make
# sense
test_template = (0., 0., 1., 0., 50.)
assert np.max(amp[test_template]) > 100. # Max amplitude is reasonable
# Average y value is about 0.
assert np.average(raw_y[test_template], weights=amp[test_template]) < 0.05
def test_full_fit():
# Finally check everything
# Create our fitter object
fitter = TemplateFitter(min_fit_pixels=0, training_library="KNN")
# Get our example data file (10 events of 1 TeV at 0 Alt, 0 Az)
data_dir = pkg_resources.resource_filename('template_builder', 'data/')
# Which needs to actually be there
data_dir += "/gamma_HESS_example.simhess.gz"
# Run full template generation
template, var_template = fitter.generate_templates(
[data_dir], "./test.template.gz", "./test_var.template.gz", True)
# Make sure we get something out
assert template is not None
assert var_template is not None
import os.path
os.path.isfile("./test.template.gz")
os.path.isfile("./test_var.template.gz")
# Open our output files
import pickle, gzip
template_fromfile = pickle.load(gzip.open("./test.template.gz", "r"))
var_template_fromfile = pickle.load(
gzip.open("./test_var.template.gz", "r"))
# And check the contents are the same
for key in template:
assert template[key].all() == template_fromfile[key].all()
assert var_template[key].all() == var_template_fromfile[key].all()
os.remove("./test.template.gz")
os.remove("./test_var.template.gz")
def test_template_fitting():
# Now lets check out our reading Chain
# Create our fitter object
fitter = TemplateFitter(min_fit_pixels=0)
# Get our example data file (10 events of 1 TeV at 0 Alt, 0 Az)
data_dir = pkg_resources.resource_filename('template_builder', 'data/')
# Which needs to actually be there
data_dir += "gamma_HESS_example.simhess.gz"
# Read in the file
amp, raw_x, raw_y = fitter.read_templates(data_dir)
test_template = (0., 0., 1., 0., 50.)
# Then lets fit our example template using the different options
fit_options = ["sklearn", "KNN"]
for option in fit_options:
fitter.training_library = option
template, var_template = fitter.fit_templates(
{test_template: amp[test_template]},
{test_template: raw_x[test_template]},
{test_template: raw_y[test_template]}, True, 1000)
assert template is not None
assert var_template is not None
x = np.linspace(fitter.bounds[0][0], fitter.bounds[0][1], fitter.bins[0])
y = np.linspace(fitter.bounds[1][0], fitter.bounds[1][1], fitter.bins[1])
# Make sure the template is the expected shape
assert template[test_template].shape[0] == fitter.bins[1]
assert template[test_template].shape[1] == fitter.bins[0]
assert var_template[test_template].shape[0] == fitter.bins[1]
assert var_template[test_template].shape[1] == fitter.bins[0]
# For now we will assume the fit just works
# Finally we will check that the range extension functions work
extended_template = fitter.extend_xmax_range(template)
xmax_range = np.array(list(extended_template.keys())).T[4]
# Check the bins are right
assert np.sort(xmax_range).all() == fitter.xmax_bins.all()
# And that all templates are the same
for key in extended_template:
assert extended_template[key].all() == template[test_template].all()
template = {test_template: template[test_template]}
# Finally check the distance extension works
template[0., 0., 1., 50., 50.] = template[test_template]
template[0., 0., 1., 100., 0.] = template[test_template]
template[0., 0., 1., 200., 0.] = template[test_template]
extended_template = fitter.extend_distance_range(template)
assert (0., 0., 1., 100., 50.) in extended_template
assert (0., 0., 1., 200., 50.) in extended_template
def get_file_list(directories):
files_before_simulation = []
for path in directories:
for file in listdir(path):
files_before_simulation.append(path + file)
return files_before_simulation
output_paths = [
'/scratch3/armstrong/LST/d2020-02-12/Data/sim_hessarray/LST/0.0deg/Data/'
]
files_after = get_file_list(output_paths)
# fitter = TemplateFitter(min_fit_pixels=2000, bounds=((-5, 1), (-1.5, 1.5)))
# fitter.generate_templates(files_after, 'test.templates.gz', max_events=50000)
fitter = TemplateFitter(min_fit_pixels=2000, verbose=True)
pool = multiprocessing.Pool(4)
templates = dict()
variance_templates = dict()
tasks = []
count = 0
for n, i in enumerate(files_after):
tasks.append([
i,
'/scratch3/armstrong/LST/d2020-02-12/Data/temp_%s.templates.gz' % n
])
count += 1
if count > 3:
break
def generate_templates():
"""
main() function to call all steps of template production in series
:return: None
"""
# First Lets parse the command line
parser = ArgumentParser()
parser.add_argument('-c', '--config', action='append', nargs=1,
metavar="config file",
help='Configuration YAML file locations')
parser.add_argument('-o', '--output', default="test.templates.gz",
metavar="output file",
help='Name of output file')
parser.add_argument('--simulate-only', dest='simulate_only', action='store_true')
parser.add_argument('--SGE', dest='SGE', action='store_true')
args = parser.parse_args()
# Followed by any config files
corsika_input, simulation_input, telescope_input, fit_input = \
parse_config(args.config)
output_file = args.output
# Generate our range of CORSIKA input cards
corsika = CORSIKAInput(input_parameters=corsika_input,
min_events=simulation_input["min_events"])
cards = corsika.get_input_cards(simulation_input["event_number"],
simulation_input["altitude"],
simulation_input["azimuth"],
simulation_input["energy_bins"],
simulation_input["core_bins"],
simulation_input["rotation_angle"],
simulation_input["diameter"],
get_run_script(telescope_input["config_name"])
)
# And write them in the sim_telarray directory
corsika_input_file_names = \
write_corsika_input_cards(telescope_input["sim_telarray_directory"], cards)
# Then create the required sim_telearray telescope config files
sim_telarray_config = SimTelArrayConfig(telescope_input["config_name"],
telescope_input["config_file"],
float(corsika_input["OBSLEV"])/100,
telescope_input["atmosphere"],
telescope_input["optical_efficiency"],
telescope_input["extra_options"]
)
run_commands, output_paths = \
sim_telarray_config.run_setup(telescope_input["sim_telarray_directory"],
corsika_input_file_names)
# Annoyingly sim_telarray doesn't let us choose our output file name (at least in
# this script setup). So we instead look in output directory now and after our
# simulations are complete and take the new files
files_before = get_file_list(output_paths)
# Submit to SGE cluster if we can
if args.SGE:
try:
from submit_SGE import SubmitSGE
except ImportError:
print("submit_SGE package required for cluster submission")
submit = SubmitSGE()
submit.submit_job_list(run_commands,
telescope_input["config_name"]+"_temp")
# Otherwise run on the command line
else:
for command in run_commands:
print("Running", command)
os.system(command)
print("Simulations complete")
files_after = get_file_list(output_paths)
# Create a list of newly created files
files_after = list(set(files_after) - set(files_before))
if len(files_after) == 0:
print("No new simulation files created! Quitting before fit")
return
# Then generate our templates from these
fitter = TemplateFitter(min_fit_pixels=2000)
fitter.generate_templates(files_after, output_file, max_events=50000)
return
def generate_templates():
"""
main() function to call all steps of template production in series
:return: None
"""
# First Lets parse the command line
parser = ArgumentParser()
parser.add_argument('-c', '--config', action='append', nargs=1,
metavar="config file",
help='Configuration YAML file locations')
parser.add_argument('-o', '--output', default="test.templates.gz",
metavar="output file",
help='Name of output file')
parser.add_argument('-m', '--cores', default=1, help='Number of cores to use for multiprocessing', type=int)
parser.add_argument('--simulate-only', dest='simulate_only', action='store_true')
parser.add_argument('--SGE', dest='SGE', action='store_true')
args = parser.parse_args()
# Followed by any config files
corsika_input, simulation_input, telescope_input, fit_input = \
parse_config(args.config)
output_file = args.output
print(corsika_input, simulation_input, telescope_input, fit_input)
# Generate our range of CORSIKA input cards
corsika = CORSIKAInput(input_parameters=corsika_input,
min_events=simulation_input["min_events"])
cards = corsika.get_input_cards(simulation_input["event_number"],
simulation_input["altitude"],
simulation_input["azimuth"],
simulation_input["energy_bins"],
simulation_input["core_bins"],
simulation_input["rotation_angle"],
simulation_input["diameter"],
get_run_script(telescope_input["config_name"])
)
# And write them in the sim_telarray directory
corsika_input_file_names = \
write_corsika_input_cards(telescope_input["sim_telarray_directory"], cards)
# Then create the required sim_telearray telescope config files
sim_telarray_config = SimTelArrayConfig(telescope_input["config_name"],
telescope_input["config_file"],
float(corsika_input["OBSLEV"]) / 100,
telescope_input["atmosphere"],
telescope_input["optical_efficiency"],
telescope_input["extra_options"]
)
run_commands, output_paths = \
sim_telarray_config.run_setup(telescope_input["sim_telarray_directory"],
corsika_input_file_names)
# Annoyingly sim_telarray doesn't let us choose our output file name (at least in
# this script setup). So we instead look in output directory now and after our
# simulations are complete and take the new files
files_before = get_file_list(output_paths)
# Submit to SGE cluster if we can
if args.SGE:
try:
from submit_SGE import SubmitSGE
except ImportError:
print("submit_SGE package required for cluster submission")
submit = SubmitSGE()
submit.submit_job_list(run_commands,
telescope_input["config_name"] + "_temp")
# Otherwise run on the command line
else:
tasks = []
print('using %s out of %s cores' % (int(args.cores), multiprocessing.cpu_count()))
pool = multiprocessing.Pool(int(args.cores))
for command in run_commands:
print("Running", command)
# exit()
tasks.append(command)
# os.system(command)
print(pool.map(os.system, tasks))
print("Simulations complete")
files_after = get_file_list(output_paths)
# Create a list of newly created files
files_after = list(set(files_after) - set(files_before))
if len(files_after) == 0:
print("No new simulation files created! Quitting before fit")
return
# tasks = []
# fitter = TemplateFitter(min_fit_pixels=2000)
#
# templates = dict()
# variance_templates = dict()
#
# for i in files_after:
# tasks.append([templates,variance_templates, i])
# # Then generate our templates from these
# pool.starmap(fitter.pool_generate_templates, tasks)
#
# fitter.pool_generate_templates(templates, variance_templates, files_after, output_file, max_events=50000)
#
# file_handler = gzip.open(output_file, "wb")
# pickle.dump(templates, file_handler)
# file_handler.close()
fitter = TemplateFitter(min_fit_pixels=2000, verbose=True)
pool = multiprocessing.Pool(int(args.cores))
templates = dict()
variance_templates = dict()
tasks = []
count = 0
for n, i in enumerate(files_after):
tasks.append([i, '/scratch3/armstrong/LST/d2020-02-12/Data/temp_%s.templates.gz' % n])
count += 1
# if count > 3:
# break
pool.starmap(fitter.pool_generate_templates, tasks)
out_dict = {}
for n, i in enumerate(files_after):
file_list = gzip.open('/scratch3/armstrong/LST/d2020-02-12/Data/temp_%d.templates.gz' % n)
input_dict = pickle.load(file_list)
out_dict.update(input_dict)
file_handler = gzip.open('/scratch3/armstrong/LST/d2020-02-12/Data/LST-LaPalma-Prod5.templates.gz', "wb")
pickle.dump(out_dict, file_handler)
file_handler.close()
return