def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper instrument module exists) and select only the
# telescopes with the same camera type
# make sure gzip files are seekable
self.reader = SimTelEventSource(
input_url=self.infile, max_events=self.max_events, back_seekable=True
)
camtypes = self.reader.subarray.to_table().group_by("camera_type")
self.reader.subarray.info(printer=self.log.info)
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group["tel_id"].data)
self._base_tel = self._selected_tels[0]
self.log.info(
"Telescope group %d: %s",
self.telgroup,
str(self.reader.subarray.tel[self._selected_tels[0]]),
)
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(parent=self, subarray=self.reader.subarray)
self.reader = SimTelEventSource(
input_url=self.infile,
max_events=self.max_events,
back_seekable=True,
allowed_tels=set(self._selected_tels),
)
def test_eventseeker():
with SimTelEventSource(input_url=dataset, back_seekable=True) as reader:
seeker = EventSeeker(event_source=reader)
event = seeker.get_event_index(1)
assert event.count == 1
event = seeker.get_event_index(0)
assert event.count == 0
event = seeker.get_event_id(31007)
assert event.index.event_id == 31007
with pytest.raises(IndexError):
seeker.get_event_index(200)
with pytest.raises(TypeError):
seeker.get_event_index("1")
with pytest.raises(TypeError):
seeker.get_event_index("t")
with pytest.raises(TypeError):
seeker.get_event_index(dict())
with SimTelEventSource(input_url=dataset, max_events=5,
back_seekable=True) as reader:
seeker = EventSeeker(event_source=reader)
with pytest.raises(IndexError):
event = seeker.get_event_index(5)
assert event is not None
def test_eventseeker():
dataset = get_dataset_path("gamma_test.simtel.gz")
kwargs = dict(config=None, tool=None, input_url=dataset)
with SimTelEventSource(**kwargs) as reader:
seeker = EventSeeker(reader=reader)
event = seeker[1]
assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119}
event = seeker[0]
assert event.r0.tels_with_data == {38, 47}
event = seeker['409']
assert event.r0.event_id == 409
assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119}
tel_list = [{38, 47}, {11, 21, 24, 26, 61, 63, 118, 119}]
events = seeker[0:2]
events_tels = [e.r0.tels_with_data for e in events]
assert events_tels == tel_list
events = seeker[[0, 1]]
events_tels = [e.r0.tels_with_data for e in events]
assert events_tels == tel_list
events = seeker[['408', '409']]
events_tels = [e.r0.tels_with_data for e in events]
assert events_tels == tel_list
assert len(seeker) == 9
with pytest.raises(IndexError):
event = seeker[200]
assert event is not None
with pytest.raises(ValueError):
event = seeker['t']
assert event is not None
with pytest.raises(TypeError):
event = seeker[dict()]
assert event is not None
with SimTelEventSource(**kwargs, max_events=5) as reader:
seeker = EventSeeker(reader=reader)
with pytest.raises(IndexError):
event = seeker[5]
assert event is not None
class StreamFileReader(SimTelEventSource):
def is_stream(self):
return True
with StreamFileReader(**kwargs) as reader:
with pytest.raises(IOError):
seeker = EventSeeker(reader=reader)
assert seeker is not None
def test_eventseeker():
with SimTelEventSource(input_url=dataset, back_seekable=True) as reader:
seeker = EventSeeker(reader=reader)
event = seeker[1]
assert event.count == 1
event = seeker[0]
assert event.count == 0
event = seeker["31007"]
assert event.index.event_id == 31007
events = seeker[0:2]
for i, event in enumerate(events):
assert event.count == i
events = seeker[[0, 1]]
for i, event in enumerate(events):
assert event.count == i
ids = ["23703", "31007"]
events = seeker[ids]
for i, event in zip(ids, events):
assert event.index.event_id == int(i)
with pytest.raises(IndexError):
event = seeker[200]
with pytest.raises(ValueError):
event = seeker["t"]
with pytest.raises(TypeError):
event = seeker[dict()]
with SimTelEventSource(
input_url=dataset, max_events=5, back_seekable=True
) as reader:
seeker = EventSeeker(reader=reader)
with pytest.raises(IndexError):
event = seeker[5]
assert event is not None
class StreamFileReader(SimTelEventSource):
def is_stream(self):
return True
with StreamFileReader(input_url=dataset) as reader:
with pytest.raises(IOError):
seeker = EventSeeker(reader=reader)
def setup(self):
# load up the telescope types table (need to first open a file, a bit of
# a hack until a proper insturment module exists) and select only the
# telescopes with the same camera type
self.reader = SimTelEventSource(
input_url=self.infile, max_events=self.max_events
)
for event in self.reader:
camtypes = event.inst.subarray.to_table().group_by('camera_type')
event.inst.subarray.info(printer=self.log.info)
break
group = camtypes.groups[self.telgroup]
self._selected_tels = list(group['id'].data)
self._base_tel = self._selected_tels[0]
self.log.info(
"Telescope group %d: %s", self.telgroup,
str(event.inst.subarray.tel[self._selected_tels[0]])
)
self.log.info(f"SELECTED TELESCOPES:{self._selected_tels}")
self.calibrator = CameraCalibrator(
parent=self, eventsource=self.reader
)
self.reader.allowed_tels = self._selected_tels
def test_comparison_to_ctapipe_true_image():
path = get_data("testing/simtel_test.simtel.gz")
source = SimTelEventSource(input_url=path)
ctapipe_images = {}
for event in source:
for telid, tel in event.mc.tel.items():
key = (event.index['event_id'], telid)
ctapipe_images[key] = tel.true_image.astype(int)
reader = SimtelReader(path,
disable_remapping=True,
only_triggered_events=True)
photoelectron_images = {}
for pe in reader:
key = (pe.metadata['event_id'], pe.metadata['telescope_id'])
photoelectron_images[key] = pe.get_photoelectrons_per_pixel(
reader.n_pixels)
assert ctapipe_images.keys() == photoelectron_images.keys()
for key in ctapipe_images.keys():
assert np.array_equal(ctapipe_images[key], photoelectron_images[key])
# Check the triggered-events are still correct when reading the non-triggered
reader = SimtelReader(path,
disable_remapping=True,
only_triggered_events=False)
photoelectron_images = {}
for pe in reader:
key = (pe.metadata['event_id'], pe.metadata['telescope_id'])
photoelectron_images[key] = pe.get_photoelectrons_per_pixel(
reader.n_pixels)
for key in ctapipe_images.keys():
assert np.array_equal(ctapipe_images[key], photoelectron_images[key])
def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
self.index = 0
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
pix_x = first_event.inst.subarray.tel[tels[0]].camera.pix_x.value
pix_y = first_event.inst.subarray.tel[tels[0]].camera.pix_y.value
self.mapping['xpix'] = pix_x
self.mapping['ypix'] = pix_y
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self.gps_time = None
self.mc_true = None
self.mc = None
self.pointing = None
self.mcheader = None
def test_eventseeker_edit():
with SimTelEventSource(input_url=dataset, back_seekable=True) as reader:
seeker = EventSeeker(event_source=reader)
event = seeker.get_event_index(1)
assert event.count == 1
event.count = 2
assert event.count == 2
event = seeker.get_event_index(1)
assert event.count == 1
def example_subarray():
"""
Subarray corresponding to the example event
"""
filename = get_dataset_path("gamma_test_large.simtel.gz")
print("******************** LOAD TEST EVENT ***********************")
with SimTelEventSource(input_url=filename) as reader:
return reader.subarray
def _global_example_event():
"""
helper to get a single event from a MC file. Don't use this fixture
directly, rather use `test_event`
"""
filename = get_dataset_path('gamma_test_large.simtel.gz')
print("******************** LOAD TEST EVENT ***********************")
with SimTelEventSource(input_url=filename) as reader:
event = next(iter(reader))
return event
def _global_example_event():
"""
helper to get a single event from a MC file. Don't use this fixture
directly, rather use `test_event`
"""
filename = get_dataset_path("gamma_test_large.simtel.gz")
print("******************** LOAD TEST EVENT ***********************")
# FIXME: switch to prod5b+ file that contains effective focal length
with SimTelEventSource(input_url=filename, focal_length_choice="nominal") as reader:
event = next(iter(reader))
return event
def test_eventseeker_simtel():
# Ensure the EventSeeker can forward seek even if back-seeking is not possible
with SimTelEventSource(input_url=dataset, back_seekable=False) as reader:
seeker = EventSeeker(event_source=reader)
event = seeker.get_event_index(1)
assert event.count == 1
event = seeker.get_event_index(1)
assert event.count == 1
event = seeker.get_event_index(2)
assert event.count == 2
event = seeker.get_event_index(2)
assert event.count == 2
event = seeker.get_event_index(4)
assert event.count == 4
with pytest.raises(IOError):
seeker.get_event_index(1)
event = seeker.get_event_index(5)
assert event.count == 5
def _subarray_and_event_gamma_off_axis_500_gev():
from ctapipe.calib import CameraCalibrator
from ctapipe.image import ImageProcessor
path = get_dataset_path("lst_prod3_calibration_and_mcphotons.simtel.zst")
with SimTelEventSource(path) as source:
it = iter(source)
# we want the second event, first event is a corner clipper
next(it)
event = next(it)
# make dl1a available
calib = CameraCalibrator(source.subarray)
calib(event)
image_processor = ImageProcessor(source.subarray)
# make dl1b available
image_processor(event)
return source.subarray, event
def test_CameraFrame_against_TelescopeFrame(filename):
input_file = get_dataset_path(filename)
# "gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz"
# )
source = SimTelEventSource(input_file,
max_events=10,
focal_length_choice="nominal")
# too few events survive for this test with the defautl quality criteria,
# use less restrictive ones
config = Config({
"StereoQualityQuery": {
"quality_criteria": [
("valid_width", "parameters.hillas.width.value > 0"),
]
}
})
calib = CameraCalibrator(subarray=source.subarray)
reconstructor = HillasReconstructor(source.subarray, config=config)
image_processor_camera_frame = ImageProcessor(source.subarray,
use_telescope_frame=False)
image_processor_telescope_frame = ImageProcessor(source.subarray,
use_telescope_frame=True)
reconstructed_events = 0
for event_telescope_frame in source:
calib(event_telescope_frame)
# make a copy of the calibrated event for the camera frame case
# later we clean and paramretrize the 2 events in the same way
# but in 2 different frames to check they return compatible results
event_camera_frame = deepcopy(event_telescope_frame)
image_processor_telescope_frame(event_telescope_frame)
image_processor_camera_frame(event_camera_frame)
result_camera_frame = reconstructor(event_camera_frame)
result_telescope_frame = reconstructor(event_telescope_frame)
assert result_camera_frame.is_valid == result_telescope_frame.is_valid
if result_telescope_frame.is_valid:
reconstructed_events += 1
for field, cam in result_camera_frame.items():
tel = getattr(result_telescope_frame, field)
if hasattr(cam, "unit"):
assert u.isclose(cam,
tel,
rtol=1e-3,
atol=1e-3 * tel.unit,
equal_nan=True)
elif isinstance(cam, list):
assert cam == tel
else:
assert np.isclose(cam,
tel,
rtol=1e-3,
atol=1e-3,
equal_nan=True)
assert reconstructed_events > 0 # check that we reconstruct at least 1 event
def test_eventsource_r1():
dataset = get_dataset_path("gamma_test_large.simtel.gz")
eventsource = SimTelEventSource(input_url=dataset)
calibrator = CameraCalibrator(eventsource=eventsource)
assert isinstance(calibrator.r1, HESSIOR1Calibrator)
import numpy as np
from matplotlib import colors, cm, pyplot as plt
from eventio import SimTelFile
from ctapipe.io import SimTelEventSource
from ctapipe.visualization import CameraDisplay
bad_pixels = {'color': 'black', 'alpha': 0.1}
cw = cm.coolwarm
cw.set_bad(**bad_pixels)
vi = cm.viridis
vi.set_bad(**bad_pixels)
p = "build/simtel-output.zst"
f = SimTelFile(p)
s = SimTelEventSource(p)
geom = s.subarray.tel[1].camera.geometry
fig, (ax_im, ax_t) = plt.subplots(ncols=2, figsize=(8, 4))
disp_im = CameraDisplay(geom, ax=ax_im, cmap=vi)
disp_im.add_colorbar()
disp_t = CameraDisplay(geom, ax=ax_t, cmap=cw)
disp_t.add_colorbar()
fig.tight_layout()
fig.show()
def plot(pe):
image = pe['photoelectrons']
from ctapipe.io import SimTelEventSource
from ctapipe.instrument import CameraGeometry
from numpy.testing import assert_allclose
events = SimTelEventSource(input_url="build/simtel-output.zst")
camera = events.subarray.tel[1].camera
geom = camera.geometry.__dict__
fact = CameraGeometry.from_name("FACT").__dict__
def test_n_pixels():
p = "n_pixels"
assert_allclose(fact[p], geom[p])
def test_camera_name():
p = "camera_name"
# assert fact[p] == geom[p]
def test_pix_id():
p = "pix_id"
assert_allclose(fact[p], geom[p])
def test_pix_x():
p = "pix_x"
assert_allclose(fact[p], geom[p])
def test_eventsource_override_r1():
dataset = get_dataset_path("gamma_test_large.simtel.gz")
eventsource = SimTelEventSource(input_url=dataset)
calibrator = CameraCalibrator(eventsource=eventsource,
r1_product="NullR1Calibrator")
assert isinstance(calibrator.r1, NullR1Calibrator)
from ctapipe.visualization import CameraDisplay
from ctapipe.io import SimTelEventSource
from eventio import SimTelFile
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from ctapipe.visualization import mpl_camera
mpl_camera.PIXEL_EPSILON = 0
path = 'build/simtel-output.zst'
# just to directly get the cam geom
subarray = SimTelEventSource(input_url=path).subarray
geom = subarray.tel[1].camera.geometry
fig, ax = plt.subplots()
im_disp = CameraDisplay(geom, ax=ax)
fig.show()
im_disp.add_colorbar()
for e in SimTelFile(path).iter_mc_events():
true_pe = e['photoelectrons'].get(0)
if true_pe is not None:
mc_shower = e['mc_shower']
energy = mc_shower['energy']
event_id = e['event_id']
def test_CameraFrame_against_TelescopeFrame(filename):
input_file = get_dataset_path(
"gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz")
source = SimTelEventSource(input_file, max_events=10)
calib = CameraCalibrator(subarray=source.subarray)
reconstructor = HillasReconstructor(source.subarray)
reconstructed_events = 0
for event in source:
calib(event)
# make a copy of the calibrated event for the camera frame case
# later we clean and paramretrize the 2 events in the same way
# but in 2 different frames to check they return compatible results
event_camera_frame = deepcopy(event)
telescope_pointings = {}
hillas_dict_camera_frame = {}
hillas_dict_telescope_frame = {}
for tel_id, dl1 in event.dl1.tel.items():
event_camera_frame.dl1.tel[
tel_id].parameters = ImageParametersContainer()
event.dl1.tel[tel_id].parameters = ImageParametersContainer()
# this is needed only here to transform the camera geometries
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=AltAz(),
)
geom_camera_frame = source.subarray.tel[tel_id].camera.geometry
# this could be done also out of this loop,
# but in case of real data each telescope would have a
# different telescope_pointing
geom_telescope_frame = geom_camera_frame.transform_to(
TelescopeFrame(telescope_pointing=telescope_pointings[tel_id]))
mask = tailcuts_clean(geom_telescope_frame,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments_camera_frame = hillas_parameters(
geom_camera_frame[mask], dl1.image[mask])
moments_telescope_frame = hillas_parameters(
geom_telescope_frame[mask], dl1.image[mask])
if (moments_camera_frame.width.value >
0) and (moments_telescope_frame.width.value > 0):
event_camera_frame.dl1.tel[
tel_id].parameters.hillas = moments_camera_frame
dl1.parameters.hillas = moments_telescope_frame
hillas_dict_camera_frame[tel_id] = moments_camera_frame
hillas_dict_telescope_frame[
tel_id] = moments_telescope_frame
else:
continue
except HillasParameterizationError as e:
print(e)
continue
if (len(hillas_dict_camera_frame) >
2) and (len(hillas_dict_telescope_frame) > 2):
reconstructor(event_camera_frame)
reconstructor(event)
reconstructed_events += 1
else: # this event was not good enough to be tested on
continue
# Compare old approach with new approach
result_camera_frame = event_camera_frame.dl2.stereo.geometry[
"HillasReconstructor"]
result_telescope_frame = event.dl2.stereo.geometry[
"HillasReconstructor"]
assert result_camera_frame.is_valid
assert result_telescope_frame.is_valid
for field in event.dl2.stereo.geometry["HillasReconstructor"].as_dict(
):
C = np.asarray(result_camera_frame.as_dict()[field])
T = np.asarray(result_telescope_frame.as_dict()[field])
assert (np.isclose(C, T, rtol=1e-03, atol=1e-03,
equal_nan=True)).all()
assert reconstructed_events > 0 # check that we reconstruct at least 1 event
def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
from ctapipe.coordinates import EngineeringCameraFrame
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
n_rows = self.mapping.metadata['n_rows']
n_columns = self.mapping.metadata['n_columns']
camera_geom = first_event.inst.subarray.tel[tels[0]].camera
engineering_frame = EngineeringCameraFrame(n_mirrors=2)
engineering_geom = camera_geom.transform_to(engineering_frame)
pix_x = engineering_geom.pix_x.value
pix_y = engineering_geom.pix_y.value
row, col = get_row_column(pix_x, pix_y)
camera_2d = np.zeros((n_rows, n_columns), dtype=np.int)
camera_2d[row, col] = np.arange(self.n_pixels, dtype=np.int)
self.pixel_order = camera_2d[self.mapping['row'], self.mapping['col']]
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self._iev = None
self._t_cpu = None
self.mc = None
self.pointing = None
self.mcheader = None
def cleanmuonwriter(muon_no, proton_no, partial_no, runno):
if not sys.warnoptions:
warnings.simplefilter("ignore")
class MultiplePeaks(Exception):
pass
class NoPeaksFound(Exception):
pass
def sig_handler(signum, frame):
print("segfault")
signal.signal(signal.SIGSEGV, sig_handler)
# Run Options
# Import raw sim_telarray output files
gamma_data = "/store/muonsims/muons3/run" + str(muon_no) + ".simtel.gz"#Muons
hadron_data = "/store/muonsims/proton6/run"+str(proton_no)+".simtel.gz"#Protons
electron_data = "/store/muonsims/partial3/run"+str(partial_no)+".simtel.gz"#Partal Muons #Change this to partial files once done
gammaflag = 0 # Should be 0 to plot muons or 1 for hadrons or 2 for partial rings. - KEEP 0 WHILST DOING FULL PROCESSING
plotev = True # Whether or not to make animation plots for one single event.
event_plot = 0 # Min event number to plot
chan = 0 # PM Channel to use.
output_filename = '/store/rogansims/cleaned' # HDF5 files output name.
# Max number of events to read in for each of gammas/protons for training.
maxcount = 2100 #Return to 2100 / 21 at end of testing
no_files = 21 # Number of files in which to store events
filerat = maxcount / no_files
print('Filerat', filerat)
no_tels = 1 # Number of telescopes
def pos_to_index(pos, size):
rnd = np.round((pos / size).to_value(unit.dimensionless_unscaled), 1)
unique = np.sort(np.unique(rnd))
mask = np.append(np.diff(unique) > 0.5, True)
bins = np.append(unique[mask] - 0.5, unique[-1] + 0.5)
return np.digitize(rnd, bins) - 1
@jit
def cam_squaremaker(data):
'''Function to translate CHEC-S integrated images into square arrays for
analysis purposes.'''
square = np.zeros(2304)
i = 0
while i < 48:
if i < 8:
xmin = 48 * i + 8
xmax = 48 * (i + 1) - 8
square[xmin:xmax] = data[i * 32:(i + 1) * 32]
i = i + 1
elif i > 7 and i < 40:
square[384:1920] = data[256:1792]
i = 40
else:
xmin = 48 * i + 8
xmax = 48 * (i + 1) - 8
square[xmin:xmax] = data[512 + i * 32:544 + i * 32]
i = i + 1
square.resize((48, 48))
square = np.flip(square, 0)
return square
cmaps = [plt.cm.jet, plt.cm.winter,
plt.cm.ocean, plt.cm.bone, plt.cm.gist_earth, plt.cm.hot,
plt.cm.cool, plt.cm.coolwarm]
hists = {}
chan = 0 # which channel to look at
count = 1 # Keeps track of number of events processed
muon_number=1
proton_number=1
partial_number=1
key=0
caliber = CameraCalibrator()
# Read in gammas/ protons from simtel for each output file.
starttime=time.time()
for fileno in np.arange(1, no_files + 1):
# Basic principle is to load in, calibrate and parameterize the gamma ray
# events, then do the same for the protons. Then mix the two together and
# write them to disk.
# Initialize lists for hdf5 storage.
to_matlab = {
'id': [],
'event_id': [],
'label': [],
'mc_energy':[],
'tel_data': [],
'tel_integrated': []}
smallfile_name = output_filename + "run" + str(runno) + "_" + str(fileno) + '.hdf5'
gamma_source = SimTelEventSource(input_url=gamma_data,back_seekable=True)
print('Processing Gammas')
evfinder = EventSeeker(reader=gamma_source)
# Determine events to load in using event seeker.
startev = int(filerat * fileno - filerat)
midev = int(filerat * fileno - filerat / 2.0)
endev = int(filerat * fileno)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber(event)
if count % 1000 == 0:
print(count)
if plotev == True and gammaflag >0:
break
to_matlab['id'].append(count)
to_matlab['event_id'].append(str(event.r0.event_id) + '01')
to_matlab['label'].append(0.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Initialize arrays for given event
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
# if plotev==True and gammaflag==0: #Plot events
# plt.imshow(squared)
# plt.show()
# plt.savefig("/home/clarkr/store/Graphs/cleanmuonplot" + str(runno) + "_" + str(muon_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
muon_number = muon_number+1
# Read in protons from simtel
print('Processing Protons')
proton_hessfile = SimTelEventSource(input_url=hadron_data,back_seekable=True)
evfinder = EventSeeker(reader=proton_hessfile)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber = CameraCalibrator()
caliber(event)
if count % 1000 == 0:
print(count)
to_matlab['id'].append(int(count))
to_matlab['event_id'].append(str(event.r0.event_id) + '02')
to_matlab['label'].append(1.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Create arrays for event.
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
#if plotev==True and gammaflag==1:
# plt.imshow(squared)
# plt.show()
# plt.savefig("/home/clarkr/store/Graphs/cleanprotonplot" + str(runno) + "_" + str(proton_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
proton_number = proton_number+1
print('Processing Partial')
electron_hessfile = SimTelEventSource(input_url=electron_data,back_seekable=True)
evfinder = EventSeeker(reader=electron_hessfile)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber = CameraCalibrator()
caliber(event)
if count % 1000 == 0:
print(count)
to_matlab['id'].append(int(count))
to_matlab['event_id'].append(str(event.r0.event_id) + '03')
to_matlab['label'].append(2.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Create arrays for event.
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
#if plotev==True and gammaflag==2:
# plt.imshow(squared)
#plt.show()
#plt.savefig("/home/clarkr/store/Graphs/cleanedpartialplot" + str(runno) + "_" + str(partial_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
partial_number = partial_number+1
# Make everything arrays in order to randomize.
to_matlab['id'] = np.asarray(to_matlab['id'])
to_matlab['event_id'] = np.asarray(to_matlab['event_id'])
to_matlab['label'] = np.asarray(to_matlab['label'])
to_matlab['mc_energy'] = np.asarray(to_matlab['mc_energy'])
to_matlab['tel_integrated'] = np.asarray(to_matlab['tel_integrated'])
no_events = len(to_matlab['label'])
randomize = np.arange(len(to_matlab['label']), dtype='int')
# Implement uniform randomization here
np.random.shuffle(randomize)
to_matlab['id'] = to_matlab['id'][randomize]
to_matlab['event_id'] = to_matlab['event_id'][randomize]
to_matlab['label'] = to_matlab['label'][randomize]
to_matlab['mc_energy'] = to_matlab['mc_energy'][randomize]
to_matlab['tel_integrated'] = to_matlab['tel_integrated'][randomize]
h5file = h5py.File(smallfile_name, "w")
print('Writing')
# HDF5 writer code
lab_event = h5file.create_dataset(
'event_label', (no_events,), dtype='i', compression="gzip")
id_group = h5file.create_dataset(
'id', (no_events,), dtype='i', compression="gzip")
id_group2 = h5file.create_dataset(
'event_id', (no_events,), dtype='i', compression="gzip")
energy_group = h5file.create_dataset(
'mc_energy', (no_events,), dtype='f', compression="gzip")
squared_group = h5file.create_dataset(
"squared_training", (no_events, 4, 48, 48), dtype='f', compression="gzip")
for index in np.arange(0, no_events):
index = int(index)
lab_event[index] = np.int64(to_matlab['label'][index])
id_group[index] = np.int64(to_matlab['id'][index])
id_group2[index] = np.int64(to_matlab['event_id'][index])
energy_group[index] = np.float64(to_matlab['mc_energy'][index])
squared_group[index, :, :, :] = to_matlab['tel_integrated'][index]
h5file.close()
endtime=time.time()
runtime=endtime-starttime
print('Time for all events to be written', runtime)
plt.show()
#plt.savefig("/home/clarkr/store/Graphs/"+str(runno)+"_"+str(gammaflag)+".png")
return
