def test_display_dl1_event(mc_gamma_testfile):
from ctapipe.io import EventSource, EventSeeker
from ctapipe.calib import CameraCalibrator
source = EventSource(mc_gamma_testfile, back_seekable=True)
seeker = EventSeeker(source)
event = seeker.get_event_index(11) # event 11 has telescopes 1 and 4 with data
CameraCalibrator(subarray=source.subarray)(event)
display_dl1_event(event, source.subarray.tel[1].camera.geometry, tel_id=1)
display_dl1_event(event, source.subarray.tel[4].camera.geometry, tel_id=4)
def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
self.index = 0
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
pix_x = first_event.inst.subarray.tel[tels[0]].camera.pix_x.value
pix_y = first_event.inst.subarray.tel[tels[0]].camera.pix_y.value
self.mapping['xpix'] = pix_x
self.mapping['ypix'] = pix_y
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self.gps_time = None
self.mc_true = None
self.mc = None
self.pointing = None
self.mcheader = None
def test_display_dl1_event():
from ctapipe.io import event_source, EventSeeker
from lstchain.tests.test_lstchain import mc_gamma_testfile
from ctapipe.calib import CameraCalibrator
source = event_source(mc_gamma_testfile, back_seekable=True)
seeker = EventSeeker(source)
event = seeker[11] # event 11 has telescopes 1 and 4 with data
CameraCalibrator()(event)
display_dl1_event(event, tel_id=1)
display_dl1_event(event, tel_id=4)
def main():
print("--> Input file: {}".format(args.input_file))
print("--> Number of events: {}".format(args.max_events))
reader = event_source(input_url=args.input_file, max_events=args.max_events)
print("--> Number of files", reader.multi_file.num_inputs())
seeker = EventSeeker(reader)
ev = seeker[0]
tel_id = ev.r0.tels_with_data[0]
n_modules = ev.lst.tel[tel_id].svc.num_modules
config = Config({
"LSTR0Corrections": {
"tel_id": tel_id
}
})
lst_r0 = LSTR0Corrections(config=config)
pedestal = DragonPedestal(tel_id=tel_id, n_module=n_modules, r0_sample_start=args.start_r0_waveform)
if args.deltaT:
print("DeltaT correction active")
for i, event in enumerate(reader):
for tel_id in event.r0.tels_with_data:
lst_r0.time_lapse_corr(event, tel_id)
pedestal.fill_pedestal_event(event)
if i%500 == 0:
print("i = {}, ev id = {}".format(i, event.r0.event_id))
else:
print("DeltaT correction no active")
for i, event in enumerate(reader):
pedestal.fill_pedestal_event(event)
if i%500 == 0:
print("i = {}, ev id = {}".format(i, event.r0.event_id))
# Finalize pedestal and write to fits file
pedestal.finalize_pedestal()
primaryhdu = fits.PrimaryHDU(ev.lst.tel[tel_id].svc.pixel_ids)
secondhdu = fits.ImageHDU(np.int16(pedestal.meanped))
hdulist = fits.HDUList([primaryhdu, secondhdu])
hdulist.writeto(args.output_file)
"--input_file",
help=
"Path to fitz.fz file to create pedestal file. Can be one path or more",
nargs='+',
type=str)
parser.add_argument("--output_file",
help="Path where script should create pedestal file",
nargs=1,
type=str)
args = parser.parse_args()
print("Input file: ", args.input_file)
print("Output file: ", args.output_file)
reader = LSTEventSource(input_url=args.input_file[0])
seeker = EventSeeker(reader)
ev = seeker[0]
n_modules = ev.lst.tel[0].svc.num_modules
ped = DragonPedestal()
PedList = []
pedestal_value_array = np.zeros((n_modules, 2, 7, 4096), dtype=np.uint16)
# create list of Pedestal class for each module
for i in range(0, n_modules):
PedList.append(DragonPedestal())
using_files_name = ''
for file_path in args.input_file:
using_files_name += (file_path.split("/")[-1])
using_files_name += ", "
def main():
description = ('Reduce a *_r1.tio file into a *_dl1.hdf5 file containing '
'various parameters extracted from the waveforms')
parser = argparse.ArgumentParser(description=description,
formatter_class=Formatter)
parser.add_argument('-f',
'--files',
dest='input_paths',
nargs='+',
help='path to the TIO r1 run files')
parser.add_argument('-m',
'--monitor',
dest='monitor',
action='store',
help='path to the monitor file (OPTIONAL)')
parser.add_argument('-o',
'--output',
dest='output_path',
action='store',
help='path to store the output HDF5 dl1 file '
'(OPTIONAL, will be automatically set if '
'not specified)')
parser.add_argument('-n',
'--maxevents',
dest='max_events',
action='store',
help='Number of events to process',
type=int)
parser.add_argument('-r',
'--reducer',
dest='reducer',
action='store',
default='AverageWF',
choices=WaveformReducerFactory.subclass_names,
help='WaveformReducer to use')
parser.add_argument('-c',
'--config',
dest='configuration',
help="""Configuration to pass to the waveform reducer
(Usage: '{"window_shift":6, "window_size":6}') """)
parser.add_argument('-p',
'--plot',
dest='plot',
action='store_true',
help="Plot stages for waveform reducers")
args = parser.parse_args()
if args.configuration:
config = json.loads(args.configuration)
config_string = args.configuration
else:
config = {}
config_string = ""
input_paths = args.input_paths
n_files = len(input_paths)
for i_path, input_path in enumerate(input_paths):
print("PROGRESS: Reducing file {}/{}".format(i_path + 1, n_files))
kwargs = dict(input_url=input_path, max_events=args.max_events)
reader = HESSIOEventSource(**kwargs)
seeker = EventSeeker(reader)
n_events = len(seeker)
first_event = seeker[0]
tels = list(first_event.r0.tels_with_data)
_, n_pixels, n_samples = first_event.r0.tel[tels[0]].waveform.shape
n_modules = 32
n_cells = 1
pixel_array = np.arange(n_pixels)
camera_version = "1.1.0"
camera_config = CameraConfiguration(camera_version)
tc_mapping = camera_config.GetMapping(n_modules == 1)
mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
if 'reference_pulse_path' not in config:
reference_pulse_path = camera_config.GetReferencePulsePath()
config['reference_pulse_path'] = reference_pulse_path
kwargs = dict(n_pixels=n_pixels,
n_samples=n_samples,
plot=args.plot,
mapping=mapping,
**config)
reducer = WaveformReducerFactory.produce(args.reducer, **kwargs)
baseline_subtractor = BaselineSubtractor(seeker)
input_path = reader.input_url
output_path = args.output_path
if not output_path:
output_path = input_path.replace(".simtel.gz", "_dl1.h5")
output_path = output_path.replace("run", "Run")
r1 = HESSIOR1Calibrator()
with DL1Writer(output_path, n_events * n_pixels,
args.monitor) as writer:
t_cpu = 0
start_time = 0
desc = "Processing events"
for event in tqdm(seeker, total=n_events, desc=desc):
iev = event.count
r1.calibrate(event)
waveforms = event.r1.tel[tels[0]].waveform[0]
mc_true = event.mc.tel[tels[0]].photo_electron_image
t_cpu = pd.to_datetime(event.trig.gps_time.value, unit='s')
if not start_time:
start_time = t_cpu
waveforms_bs = baseline_subtractor.subtract(waveforms)
bs = baseline_subtractor.baseline
params = reducer.process(waveforms_bs)
df_ev = pd.DataFrame(
dict(iev=iev,
pixel=pixel_array,
first_cell_id=0,
t_cpu=t_cpu,
t_tack=0,
baseline_subtracted=bs,
**params,
mc_true=mc_true))
writer.append_event(df_ev)
sn_dict = {}
for tm in range(n_modules):
sn_dict['TM{:02d}_SN'.format(tm)] = "NaN"
metadata = dict(source="CHECLabPy",
date_generated=pd.datetime.now(),
input_path=input_path,
n_events=n_events,
n_modules=n_modules,
n_pixels=n_pixels,
n_samples=n_samples,
n_cells=n_cells,
start_time=start_time,
end_time=t_cpu,
camera_version=camera_version,
reducer=reducer.__class__.__name__,
configuration=config_string,
**sn_dict)
writer.add_metadata(**metadata)
writer.add_mapping(mapping)
def cleanmuonwriter(muon_no, proton_no, partial_no, runno):
if not sys.warnoptions:
warnings.simplefilter("ignore")
class MultiplePeaks(Exception):
pass
class NoPeaksFound(Exception):
pass
def sig_handler(signum, frame):
print("segfault")
signal.signal(signal.SIGSEGV, sig_handler)
# Run Options
# Import raw sim_telarray output files
gamma_data = "/store/muonsims/muons3/run" + str(muon_no) + ".simtel.gz"#Muons
hadron_data = "/store/muonsims/proton6/run"+str(proton_no)+".simtel.gz"#Protons
electron_data = "/store/muonsims/partial3/run"+str(partial_no)+".simtel.gz"#Partal Muons #Change this to partial files once done
gammaflag = 0 # Should be 0 to plot muons or 1 for hadrons or 2 for partial rings. - KEEP 0 WHILST DOING FULL PROCESSING
plotev = True # Whether or not to make animation plots for one single event.
event_plot = 0 # Min event number to plot
chan = 0 # PM Channel to use.
output_filename = '/store/rogansims/cleaned' # HDF5 files output name.
# Max number of events to read in for each of gammas/protons for training.
maxcount = 2100 #Return to 2100 / 21 at end of testing
no_files = 21 # Number of files in which to store events
filerat = maxcount / no_files
print('Filerat', filerat)
no_tels = 1 # Number of telescopes
def pos_to_index(pos, size):
rnd = np.round((pos / size).to_value(unit.dimensionless_unscaled), 1)
unique = np.sort(np.unique(rnd))
mask = np.append(np.diff(unique) > 0.5, True)
bins = np.append(unique[mask] - 0.5, unique[-1] + 0.5)
return np.digitize(rnd, bins) - 1
@jit
def cam_squaremaker(data):
'''Function to translate CHEC-S integrated images into square arrays for
analysis purposes.'''
square = np.zeros(2304)
i = 0
while i < 48:
if i < 8:
xmin = 48 * i + 8
xmax = 48 * (i + 1) - 8
square[xmin:xmax] = data[i * 32:(i + 1) * 32]
i = i + 1
elif i > 7 and i < 40:
square[384:1920] = data[256:1792]
i = 40
else:
xmin = 48 * i + 8
xmax = 48 * (i + 1) - 8
square[xmin:xmax] = data[512 + i * 32:544 + i * 32]
i = i + 1
square.resize((48, 48))
square = np.flip(square, 0)
return square
cmaps = [plt.cm.jet, plt.cm.winter,
plt.cm.ocean, plt.cm.bone, plt.cm.gist_earth, plt.cm.hot,
plt.cm.cool, plt.cm.coolwarm]
hists = {}
chan = 0 # which channel to look at
count = 1 # Keeps track of number of events processed
muon_number=1
proton_number=1
partial_number=1
key=0
caliber = CameraCalibrator()
# Read in gammas/ protons from simtel for each output file.
starttime=time.time()
for fileno in np.arange(1, no_files + 1):
# Basic principle is to load in, calibrate and parameterize the gamma ray
# events, then do the same for the protons. Then mix the two together and
# write them to disk.
# Initialize lists for hdf5 storage.
to_matlab = {
'id': [],
'event_id': [],
'label': [],
'mc_energy':[],
'tel_data': [],
'tel_integrated': []}
smallfile_name = output_filename + "run" + str(runno) + "_" + str(fileno) + '.hdf5'
gamma_source = SimTelEventSource(input_url=gamma_data,back_seekable=True)
print('Processing Gammas')
evfinder = EventSeeker(reader=gamma_source)
# Determine events to load in using event seeker.
startev = int(filerat * fileno - filerat)
midev = int(filerat * fileno - filerat / 2.0)
endev = int(filerat * fileno)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber(event)
if count % 1000 == 0:
print(count)
if plotev == True and gammaflag >0:
break
to_matlab['id'].append(count)
to_matlab['event_id'].append(str(event.r0.event_id) + '01')
to_matlab['label'].append(0.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Initialize arrays for given event
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
# if plotev==True and gammaflag==0: #Plot events
# plt.imshow(squared)
# plt.show()
# plt.savefig("/home/clarkr/store/Graphs/cleanmuonplot" + str(runno) + "_" + str(muon_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
muon_number = muon_number+1
# Read in protons from simtel
print('Processing Protons')
proton_hessfile = SimTelEventSource(input_url=hadron_data,back_seekable=True)
evfinder = EventSeeker(reader=proton_hessfile)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber = CameraCalibrator()
caliber(event)
if count % 1000 == 0:
print(count)
to_matlab['id'].append(int(count))
to_matlab['event_id'].append(str(event.r0.event_id) + '02')
to_matlab['label'].append(1.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Create arrays for event.
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
#if plotev==True and gammaflag==1:
# plt.imshow(squared)
# plt.show()
# plt.savefig("/home/clarkr/store/Graphs/cleanprotonplot" + str(runno) + "_" + str(proton_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
proton_number = proton_number+1
print('Processing Partial')
electron_hessfile = SimTelEventSource(input_url=electron_data,back_seekable=True)
evfinder = EventSeeker(reader=electron_hessfile)
print(startev, endev)
for event in evfinder[startev:endev]:
caliber = CameraCalibrator()
caliber(event)
if count % 1000 == 0:
print(count)
to_matlab['id'].append(int(count))
to_matlab['event_id'].append(str(event.r0.event_id) + '03')
to_matlab['label'].append(2.0)
energy=event.mc.energy.to(unit.GeV)
print(energy.value)
to_matlab['mc_energy'].append(energy.value)
# Create arrays for event.
integrated = np.zeros((no_tels, 48, 48))
for index, tel_id in enumerate(event.dl0.tels_with_data):
# Loop through all triggered telescopes.
cam = event.inst.subarray.tel[tel_id].camera
size = np.sqrt(cam.pix_area)
col = pos_to_index(cam.pix_x, size)
row = pos_to_index(cam.pix_y, size)
integ_charges = event.dl1.tel[tel_id]['image']
#ImageCleaningSection - From CTA documentation
cleanmask=tailcuts_clean(cam, integ_charges, picture_thresh=1.7, boundary_thresh=1.5, min_number_picture_neighbors=2)
cleaned = integ_charges.copy()
cleaned[~cleanmask] = 0
squared = np.full((row.max() + 1, col.max() + 1), np.nan)
squared[row, col] = cleaned
integrated[tel_id - 1, :, :] = squared
#if plotev==True and gammaflag==2:
# plt.imshow(squared)
#plt.show()
#plt.savefig("/home/clarkr/store/Graphs/cleanedpartialplot" + str(runno) + "_" + str(partial_number) + ".png")
# Send to hdf5 writer lists.
# List of triggered telescopes
to_matlab['tel_integrated'].append(integrated)
count = count + 1
partial_number = partial_number+1
# Make everything arrays in order to randomize.
to_matlab['id'] = np.asarray(to_matlab['id'])
to_matlab['event_id'] = np.asarray(to_matlab['event_id'])
to_matlab['label'] = np.asarray(to_matlab['label'])
to_matlab['mc_energy'] = np.asarray(to_matlab['mc_energy'])
to_matlab['tel_integrated'] = np.asarray(to_matlab['tel_integrated'])
no_events = len(to_matlab['label'])
randomize = np.arange(len(to_matlab['label']), dtype='int')
# Implement uniform randomization here
np.random.shuffle(randomize)
to_matlab['id'] = to_matlab['id'][randomize]
to_matlab['event_id'] = to_matlab['event_id'][randomize]
to_matlab['label'] = to_matlab['label'][randomize]
to_matlab['mc_energy'] = to_matlab['mc_energy'][randomize]
to_matlab['tel_integrated'] = to_matlab['tel_integrated'][randomize]
h5file = h5py.File(smallfile_name, "w")
print('Writing')
# HDF5 writer code
lab_event = h5file.create_dataset(
'event_label', (no_events,), dtype='i', compression="gzip")
id_group = h5file.create_dataset(
'id', (no_events,), dtype='i', compression="gzip")
id_group2 = h5file.create_dataset(
'event_id', (no_events,), dtype='i', compression="gzip")
energy_group = h5file.create_dataset(
'mc_energy', (no_events,), dtype='f', compression="gzip")
squared_group = h5file.create_dataset(
"squared_training", (no_events, 4, 48, 48), dtype='f', compression="gzip")
for index in np.arange(0, no_events):
index = int(index)
lab_event[index] = np.int64(to_matlab['label'][index])
id_group[index] = np.int64(to_matlab['id'][index])
id_group2[index] = np.int64(to_matlab['event_id'][index])
energy_group[index] = np.float64(to_matlab['mc_energy'][index])
squared_group[index, :, :, :] = to_matlab['tel_integrated'][index]
h5file.close()
endtime=time.time()
runtime=endtime-starttime
print('Time for all events to be written', runtime)
plt.show()
#plt.savefig("/home/clarkr/store/Graphs/"+str(runno)+"_"+str(gammaflag)+".png")
return
def __init__(self, path, max_events=None):
"""
Reads simtelarray files utilising the SimTelEventSource from ctapipe
Parameters
----------
path : str
Path to the simtel file
max_events : int
Maximum number of events to read from the file
"""
super().__init__(path, max_events)
try:
from ctapipe.io import SimTelEventSource, EventSeeker
from ctapipe.coordinates import EngineeringCameraFrame
except ModuleNotFoundError:
msg = "Cannot find ctapipe installation"
raise ModuleNotFoundError(msg)
try:
from target_calib import CameraConfiguration
except ModuleNotFoundError:
msg = ("Cannot find TARGET libraries, please follow installation "
"instructions from https://forge.in2p3.fr/projects/gct/"
"wiki/Installing_CHEC_Software")
raise ModuleNotFoundError(msg)
self.path = path
reader = SimTelEventSource(input_url=path,
max_events=max_events,
back_seekable=True)
self.seeker = EventSeeker(reader)
first_event = self.seeker[0]
tels = list(first_event.r0.tels_with_data)
self.tel = tels[0]
shape = first_event.r0.tel[self.tel].waveform.shape
_, self.n_pixels, self.n_samples = shape
self.n_modules = self.n_pixels // 64
n_modules = 32
camera_version = "1.1.0"
self._camera_config = CameraConfiguration(camera_version)
tc_mapping = self._camera_config.GetMapping(n_modules == 1)
self.mapping = get_clp_mapping_from_tc_mapping(tc_mapping)
n_rows = self.mapping.metadata['n_rows']
n_columns = self.mapping.metadata['n_columns']
camera_geom = first_event.inst.subarray.tel[tels[0]].camera
engineering_frame = EngineeringCameraFrame(n_mirrors=2)
engineering_geom = camera_geom.transform_to(engineering_frame)
pix_x = engineering_geom.pix_x.value
pix_y = engineering_geom.pix_y.value
row, col = get_row_column(pix_x, pix_y)
camera_2d = np.zeros((n_rows, n_columns), dtype=np.int)
camera_2d[row, col] = np.arange(self.n_pixels, dtype=np.int)
self.pixel_order = camera_2d[self.mapping['row'], self.mapping['col']]
self.reference_pulse_path = self._camera_config.GetReferencePulsePath()
self.camera_version = self._camera_config.GetVersion()
self._iev = None
self._t_cpu = None
self.mc = None
self.pointing = None
self.mcheader = None
