def __init__(self, n_pixels, n_samples, **kwargs):
super().__init__(n_pixels, n_samples, **kwargs)
try:
from ctapipe.image.extractor import LocalPeakWindowSum
except ImportError:
msg = ("ctapipe not found. Please either install ctapipe or "
"disable the columns from WaveformReducer {} ({})"
.format(self.__class__.__name__, self.columns))
raise ImportError(msg)
self.window_size = self.kwargs.get("window_size", 6)
self.window_shift = self.kwargs.get("window_shift", 3)
self.integrator = LocalPeakWindowSum(
window_shift=self.window_shift,
window_width=self.window_size
)
def test_config(example_subarray):
window_shift = 3
window_width = 9
config = Config(
{
"LocalPeakWindowSum": {
"window_shift": window_shift,
"window_width": window_width,
}
}
)
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=LocalPeakWindowSum(subarray=example_subarray, config=config),
config=config,
)
assert calibrator.image_extractor.window_shift.tel[None] == window_shift
assert calibrator.image_extractor.window_width.tel[None] == window_width
class ArrivalTimeCorr:
extractor = LocalPeakWindowSum()
def __init__(self, fan_array, fbn_array, n_harm, offset=380):
self.fan_array = fan_array
self.fbn_array = fbn_array
self.n_harm = n_harm
self.offset = offset
self.arrival_time_list = [[] for i in range(1855)]
self.arrival_time_corr_list = [[] for i in range(1855)]
def corr_arrivial_time(self, ev, N_module=265):
expected_pixel_id = ev.lst.tel[0].svc.pixel_ids
baseline_subtracted = ev.r1.tel[0].waveform[:, :, 2:38] - 380
try:
charge, pulse_time = self.extractor(baseline_subtracted)
pulse_time = extract_pulse_time(baseline_subtracted[0, :, :]) + 2
for nr in prange(0, N_module):
fc = get_first_capacitor(ev.lst.tel[0].evt.first_capacitor_id,
nr)
for pix in prange(0, 7):
pixel = expected_pixel_id[nr * 7 + pix]
if charge[0, pixel] > 1500:
self.arrival_time_list[pixel].append(pulse_time[pixel])
corr_pos = get_corr_time(fc[0, pix] % 1024,
self.fan_array[pixel],
self.fbn_array[pixel],
fNumHarmonics=self.n_harm)
corr_time = pulse_time[
pixel] - corr_pos #+ get_mean_time(self.fan_array[pixel])
self.arrival_time_corr_list[pixel].append(corr_time)
except ZeroDivisionError:
pass
def get_arrivial_time_list(self):
return self.arrival_time_list
def get_arrivial_time_corr_list(self):
return self.arrival_time_corr_list
def test_local_peak_window_sum(camera_waveforms):
waveforms, _ = camera_waveforms
extractor = LocalPeakWindowSum()
charge, pulse_time = extractor(waveforms)
assert_allclose(charge[0], 240.3, rtol=1e-3)
assert_allclose(pulse_time[0], 46.036266, rtol=1e-3)
def test_manual_extractor():
calibrator = CameraCalibrator(image_extractor=LocalPeakWindowSum())
assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
def test_manual_extractor(example_subarray):
calibrator = CameraCalibrator(
subarray=example_subarray,
image_extractor=LocalPeakWindowSum(subarray=example_subarray))
assert isinstance(calibrator.image_extractor, LocalPeakWindowSum)
class PulseCorrection:
extractor = LocalPeakWindowSum()
def __init__(self, fan_array, fbn_array, n_harm, offset=380):
self.fan_array = fan_array
self.fbn_array = fbn_array
self.n_harm = n_harm
self.offset = offset
self.mean_time_list = []
self.raw_pulse_list = [[] for i in range(1855)]
self.corr_pulse_list = [[] for i in range(1855)]
self.corr_mean_pulse_list = [[] for i in range(1855)]
def corr_pulse(self, ev):
expected_pixel_id = ev.lst.tel[0].svc.pixel_ids
baseline_subtracted = ev.r1.tel[0].waveform[:, :, 2:38] - 380
try:
charge, _ = self.extractor(baseline_subtracted)
pulse_time = extract_pulse_time(baseline_subtracted[0, :, :])
corr_time_list = []
for nr in prange(0, 265):
fc = get_first_capacitor(ev, nr)
for pix in prange(0, 7):
pixel = expected_pixel_id[nr * 7 + pix]
if charge[0, pixel] > 1500:
corr_pos = get_corr_time(fc[0, pix] % 1024,
self.fan_array[pixel],
self.fbn_array[pixel],
fNumHarmonics=self.n_harm)
corr_time = pulse_time[pixel] - corr_pos
corr_time_list.append(corr_time)
if len(corr_time_list) > 1500:
mean_time = np.nanmean(corr_time_list)
self.mean_time_list.append(mean_time)
for nr in prange(0, 265):
fc = get_first_capacitor(ev, nr)
for pix in prange(0, 7):
pixel = expected_pixel_id[nr * 7 + pix]
if charge[0, pixel] > 1500:
corr_pos = get_corr_time(fc[0, pix] % 1024,
self.fan_array[pixel],
self.fbn_array[pixel],
fNumHarmonics=self.n_harm)
corr_time = pulse_time[pixel] - corr_pos
dt = corr_time - mean_time
self.raw_pulse_list[pixel].append(
pulse_time[pixel])
self.corr_pulse_list[pixel].append(corr_time)
self.corr_mean_pulse_list[pixel].append(dt)
except Exception as err:
print(err)
def get_raw_pulse_list(self):
return self.raw_pulse_list
def get_corr_pulse_list(self):
return self.corr_pulse_list
def get_corr_mean_pulse_list(self):
return self.corr_mean_pulse_list
def get_events(filename,
storedata=False,
test=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Depreciated, use r0_to_dl1.
Read a Simtelarray file, extract pixels charge, calculate image
parameters and timing parameters and store the result in an hdf5
file.
Parameters:
-----------
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
Returns:
--------
pandas DataFrame: output
"""
from warnings import warn
warn("Deprecated: use r0_to_dl1")
#Particle type:
particle_type = utils.guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'obs_id',
'event_id',
'mc_energy',
'mc_alt',
'mc_az',
'mc_core_x',
'mc_core_y',
'mc_h_first_int',
'mc_type',
'gps_time',
'width',
'length',
'wl',
'phi',
'psi',
'r',
'x',
'y',
'intensity',
'skewness',
'kurtosis',
'mc_alt_tel',
'mc_az_tel',
'mc_core_distance',
'mc_x_max',
'time_gradient',
'intercept',
'src_x',
'src_y',
'disp_norm',
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = event_source(input_url=filename,
allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
if test == True and i > 1000: # for quick tests
break
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal # the pedestal is the
#average (for pedestal events) of the *sum* of all samples,
#from sim_telarray
nsamples = data.shape[2] # total number of samples
# Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples
integrator = LocalPeakWindowSum()
integration, pulse_time = integrator(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the
# corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(np.log10(signals))
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(cleanmask)
clean = signals.copy()
clean[~cleanmask] = 0.0 # set to 0 pixels which did not
# survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(geom, clean)
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom, clean, peak_time, hillas)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
#Hillas parameters
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#MC data:
obs_id = event.r0.obs_id
event_id = event.r0.event_id
mc_energy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mc_alt = event.mc.alt.value
mc_az = event.mc.az.value
mc_core_x = event.mc.core_x.value
mc_core_y = event.mc.core_y.value
mc_h_first_int = event.mc.h_first_int.value
mc_type = event.mc.shower_primary_id
mc_az_tel = event.mcheader.run_array_direction[0].value
mc_alt_tel = event.mcheader.run_array_direction[1].value
mc_x_max = event.mc.x_max.value
gps_time = event.trig.gps_time.value
#Calculate mc_core_distance parameters
mc_core_distance = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
#Timing parameters
time_gradient = timepars['slope'].value
intercept = timepars['intercept']
#Calculate disp_ and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel,
mc_az_tel, focal_length)
src_x = sourcepos[0]
src_y = sourcepos[1]
disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y)
eventdf = pd.DataFrame([[
obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x,
mc_core_y, mc_h_first_int, mc_type, gps_time, width,
length, width / length, phi, psi, r, x, y, intensity,
skewness, kurtosis, mc_alt_tel, mc_az_tel,
mc_core_distance, mc_x_max, time_gradient, intercept,
src_x, src_y, disp, mc_type
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if (concatenate == False
or (concatenate == True
and np.DataSource().exists(outfile) == False)):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
"""Initiliaze an EventPreparer object."""
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meters
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)),
# TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946))
# in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
extractor = LocalPeakWindowSum(config=cfg)
self.calib = CameraCalibrator(config=cfg, image_extractor=extractor)
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
def get_first_capacitor(event, nr, tel_id):
high_gain = 0
low_gain = 1
fc = np.zeros((2, 7))
first_cap = event.lst.tel[tel_id].evt.first_capacitor_id[nr * 8:(nr + 1) *
8]
# First capacitor order according Dragon v5 board data format
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [0, 0, 1, 1, 2, 2, 3]):
fc[high_gain, i] = first_cap[j]
for i, j in zip([0, 1, 2, 3, 4, 5, 6], [4, 4, 5, 5, 6, 6, 7]):
fc[low_gain, i] = first_cap[j]
return fc
extractor = LocalPeakWindowSum()
def get_pulse_before_and_after_time_corr(event):
gain = 0
tel_id = 1
n_harm = 16
pixel_ids = ev.lst.tel[tel_id].svc.pixel_ids
time_list = []
time_corr_list = []
time_corr_mean_list = []
for nr in range(0, 265):
for pix in range(0, 7):
fc = get_first_capacitor(event, nr, tel_id)[gain, pix]
def main():
print("input file: {}".format(args.input_file))
print("output file: {}".format(args.output_file))
print("pedestal file: {}".format(args.pedestal_file))
print("calibration file: {}".format(args.calibration_file))
print("max events: {}".format(args.max_events))
# Camera geometry
geom = CameraGeometry.from_name("LSTCam-002")
# Definition of the output parameters for the table
output_parameters = {
'event_id': [],
'ring_size': [],
'size_outside': [],
'ring_radius': [],
'ring_width': [],
'good_ring': [],
'muon_efficiency': [],
'ring_containment': [],
'ring_completeness': [],
'ring_pixel_completeness': [],
'impact_parameter': [],
'impact_x_array': [],
'impact_y_array': [],
}
# Calibration related quantities
r0calib = LSTR0Corrections(pedestal_path=args.pedestal_file,
r1_sample_start=2,
r1_sample_end=38)
ff_data = FlatFieldContainer()
cal_data = WaveformCalibrationContainer()
ped_data = PedestalContainer()
dc_to_pe = []
ped_median = []
if (args.run_number >
500): # Not sure where did the tel definition change
with HDF5TableReader(args.calibration_file) as h5_table:
assert h5_table._h5file.isopen == True
for cont in h5_table.read('/tel_1/pedestal', ped_data):
ped_median = cont.charge_median
for calib in h5_table.read('/tel_1/calibration', cal_data):
dc_to_pe = calib['dc_to_pe']
h5_table.close()
else:
with HDF5TableReader(args.calibration_file) as h5_table:
assert h5_table._h5file.isopen == True
for cont in h5_table.read('/tel_0/pedestal', ped_data):
ped_median = cont.charge_median
for calib in h5_table.read('/tel_0/calibration', cal_data):
dc_to_pe = calib['dc_to_pe']
h5_table.close()
# Maximum number of events
if (args.max_events):
max_events = args.max_events
else:
max_events = None
# File open
num_muons = 0
source = event_source(input_url=args.input_file, max_events=max_events)
for event in source:
r0calib.calibrate(event)
# Not sure where did the tel definition change
# but we moved to tel[0] to tel[1] at some point
# of the commissioning period
if (args.run_number > 500):
event_id = event.lst.tel[1].evt.event_id
telescope_description = event.inst.subarray.tel[1]
pedcorrectedsamples = event.r1.tel[1].waveform
else:
event_id = event.lst.tel[0].evt.event_id
telescope_description = event.inst.subarray.tel[0]
pedcorrectedsamples = event.r1.tel[0].waveform
integrator = LocalPeakWindowSum(window_shift=4, window_width=9)
integration, pulse_time = integrator(pedcorrectedsamples)
image = (integration - ped_median) * dc_to_pe
# WARNING!!!
# The current analysis is not performed using gain selection
# image[0] is the extracted image from low gain.
print("Event {}. Number of pixels above 10 phe: {}".format(
event_id, np.size(image[0][image[0] > 10.])))
if not tag_pix_thr(
image[0]): #default skipps pedestal and calibration events
continue
if not muon_filter(image[0]): #default values apply no filtering
continue
equivalent_focal_length = telescope_description.optics.equivalent_focal_length
mirror_area = telescope_description.optics.mirror_area.to("m2")
muonintensityparam, size_outside_ring, muonringparam, good_ring = \
analyze_muon_event(event_id, image[0], geom, equivalent_focal_length,
mirror_area, args.plot_rings, args.plots_path)
#if not (good_ring):
# continue
print("Number of muons found {}, EventID {}".format(
num_muons, event_id))
num_muons = num_muons + 1
output_parameters['event_id'].append(event_id)
output_parameters['ring_size'].append(muonintensityparam.ring_size)
output_parameters['size_outside'].append(size_outside_ring)
output_parameters['ring_radius'].append(
muonringparam.ring_radius.value)
output_parameters['ring_width'].append(
muonintensityparam.ring_width.value)
output_parameters['good_ring'].append(good_ring)
output_parameters['muon_efficiency'].append(
muonintensityparam.optical_efficiency_muon)
output_parameters['ring_containment'].append(
muonringparam.ring_containment)
output_parameters['ring_completeness'].append(
muonintensityparam.ring_completeness)
output_parameters['ring_pixel_completeness'].append(
muonintensityparam.ring_pix_completeness)
output_parameters['impact_parameter'].append(
muonintensityparam.impact_parameter.value)
output_parameters['impact_x_array'].append(
muonintensityparam.impact_parameter_pos_x.value)
output_parameters['impact_y_array'].append(
muonintensityparam.impact_parameter_pos_y.value)
table = Table(output_parameters)
if os.path.exists(args.output_file):
os.remove(args.output_file)
table.write(args.output_file, format='fits')
class TimeCalCorr:
extractor = LocalPeakWindowSum()
def __init__(self, n_combine, n_harm, n_cap):
n = int(n_cap / n_combine)
self.fMeanVal = np.zeros((1855, n))
self.fNumMean = np.zeros((1855, n))
self.fNumCombine = n_combine
self.fNumHarmonics = n_harm
self.fNumCap = n_cap
self.fNumPoints = int(self.fNumCap / self.fNumCombine)
self.first_cap_array = np.zeros((265, 2, 7))
self.n_modules = 265
def calib_pulse_time(self, ev):
pixel_ids = ev.lst.tel[0].svc.pixel_ids
get_first_capacitor_jit(ev.lst.tel[0].evt.first_capacitor_id,
self.first_cap_array)
try:
baseline_subtracted = ev.r1.tel[0].waveform[:, :, 2:38] - 380
charge, _ = self.extractor(baseline_subtracted)
pulse_time = extract_pulse_time(baseline_subtracted[0, :, :]) + 2
fill_jit(0,
pixel_ids,
self.first_cap_array,
charge,
pulse_time,
self.fMeanVal,
self.fNumMean,
fNumCombine=self.fNumCombine)
except ZeroDivisionError:
pass
def fill(self, gain, pixel_ids, first_cap_array, integration, pulse_time):
for nr in range(0, 265):
fc = get_first_capacitor(first_cap_array, nr)
for pix in range(0, 7):
pixel = pixel_ids[nr * 7 + pix]
if integration[gain, pixel] > 3000:
first_cap = fc[gain, pix] % 1024
fBin = int(first_cap / self.fNumCombine)
self.fMeanVal[pixel, fBin] += pulse_time[pixel]
self.fNumMean[pixel, fBin] += 1
def finalize(self):
self.fMeanVal = self.fMeanVal / self.fNumMean
def fit(self, pixel_id):
self.pos = np.zeros(self.fNumPoints)
for i in range(0, self.fNumPoints):
self.pos[i] = (i + 0.5) * self.fNumCombine
self.fan = np.zeros(self.fNumHarmonics)
self.fbn = np.zeros(self.fNumHarmonics)
for n in range(0, self.fNumHarmonics):
self.integrate_with_trig(self.pos, self.fMeanVal[pixel_id], n,
self.fan, self.fbn)
def integrate_with_trig(self, x, y, n, an, bn):
suma = 0
sumb = 0
for i in range(0, self.fNumPoints):
suma += y[i] * self.fNumCombine * np.cos(
2 * np.pi * n * (x[i] / float(self.fNumCap)))
sumb += y[i] * self.fNumCombine * np.sin(
2 * np.pi * n * (x[i] / float(self.fNumCap)))
an[n] = suma * (2. / (self.fNumPoints * self.fNumCombine))
bn[n] = sumb * (2. / (self.fNumPoints * self.fNumCombine))