def test_grad_fit():
"""
Simple test that gradient fitting gives expected answers for perfect
gradient
"""
grad = 2
intercept = 1
timing = timing_parameters(
pix_x=np.zeros(4) * u.deg,
pix_y=np.arange(4) * u.deg,
image=np.ones(4),
peak_time=intercept * u.ns + grad * np.arange(4) * u.ns,
rotation_angle=0 * u.deg
)
# Test we get the values we put in back out again
assert_allclose(timing.gradient, grad * u.ns / u.deg)
assert_allclose(timing.intercept, intercept * u.deg)
# Then try a different rotation angle
rot_angle = 20 * u.deg
timing_rot20 = timing_parameters(
pix_x=np.zeros(4) * u.deg,
pix_y=np.arange(4) * u.deg,
image=np.ones(4),
peak_time=intercept * u.ns +
grad * np.arange(4) * u.ns,
rotation_angle=rot_angle
)
# Test the output again makes sense
assert_allclose(timing_rot20.gradient, timing.gradient / np.cos(rot_angle))
assert_allclose(timing_rot20.intercept, timing.intercept)
def test_psi_0():
"""
Simple test that gradient fitting gives expected answers for perfect
gradient
"""
grad = 2.0
intercept = 1.0
deviation = 0.1
geom = CameraGeometry.from_name("LSTCam")
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg)
random = np.random.RandomState(1)
pulse_time = intercept + grad * geom.pix_x.value
pulse_time += random.normal(0, deviation, geom.n_pixels)
timing = timing_parameters(geom,
image=np.ones(geom.n_pixels),
pulse_time=pulse_time,
hillas_parameters=hillas,
cleaning_mask=np.ones(geom.n_pixels,
dtype=bool))
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2)
assert_allclose(timing.intercept, intercept, rtol=1e-2)
assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_psi_20():
# Then try a different rotation angle
grad = 2
intercept = 1
deviation = 0.1
geom = CameraGeometry.from_name("LSTCam")
psi = 20 * u.deg
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=psi)
random = np.random.RandomState(1)
pulse_time = intercept + grad * (np.cos(psi) * geom.pix_x.value +
np.sin(psi) * geom.pix_y.value)
pulse_time += random.normal(0, deviation, geom.n_pixels)
timing = timing_parameters(geom,
image=np.ones(geom.n_pixels),
pulse_time=pulse_time,
hillas_parameters=hillas,
cleaning_mask=np.ones(geom.n_pixels,
dtype=bool))
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2)
assert_allclose(timing.intercept, intercept, rtol=1e-2)
assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_ignore_negative():
grad = 2.0
intercept = 1.0
deviation = 0.1
geom = CameraGeometry.from_name("LSTCam")
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg)
random = np.random.RandomState(1)
pulse_time = intercept + grad * geom.pix_x.value
pulse_time += random.normal(0, deviation, geom.n_pixels)
image = np.ones(geom.n_pixels)
image[5:10] = -1.0
cleaning_mask = image >= 0
timing = timing_parameters(
geom,
image,
pulse_time=pulse_time,
hillas_parameters=hillas,
cleaning_mask=cleaning_mask,
)
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit, rtol=1e-2)
assert_allclose(timing.intercept, intercept, rtol=1e-2)
assert_allclose(timing.deviation, deviation, rtol=1e-2)
def test_array_display():
""" check that we can do basic array display functionality """
from ctapipe.visualization.mpl_array import ArrayDisplay
from ctapipe.image.timing_parameters import timing_parameters
# build a test subarray:
tels = dict()
tel_pos = dict()
for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m):
tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam")
tel_pos[ii + 1] = pos
sub = SubarrayDescription(name="TestSubarray",
tel_positions=tel_pos,
tel_descriptions=tels)
ad = ArrayDisplay(sub)
ad.set_vector_rho_phi(1 * u.m, 90 * u.deg)
# try setting a value
vals = ones(sub.num_tels)
ad.values = vals
assert (vals == ad.values).all()
# test using hillas params:
hillas_dict = {
1: HillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg),
2: HillasParametersContainer(length=20000 * u.cm, psi="95deg"),
}
grad = 2
intercept = 1
geom = CameraGeometry.from_name("LSTCam")
rot_angle = 20 * u.deg
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=rot_angle)
timing_rot20 = timing_parameters(
geom,
image=ones(geom.n_pixels),
pulse_time=intercept + grad * geom.pix_x.value,
hillas_parameters=hillas,
cleaning_mask=ones(geom.n_pixels, dtype=bool),
)
gradient_dict = {
1: timing_rot20.slope.value,
2: timing_rot20.slope.value,
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
ad.set_line_hillas(hillas_dict, range=300)
ad.add_labels()
ad.remove_labels()
def set_timing_features(self, geom, image, pulse_time, hillas):
try: # if np.polyfit fails (e.g. len(image) < deg + 1)
timepars = time.timing_parameters(geom, image, pulse_time, hillas)
self.time_gradient = timepars.slope.value
self.intercept = timepars.intercept
except ValueError:
self.time_gradient = np.nan
self.intercept = np.nan
def set_timing_features(self, geom, image, peakpos, hillas):
try:
peak_time = Quantity(peakpos) * u.Unit("ns")
timepars = time.timing_parameters(geom, image, peak_time, hillas)
self.time_gradient = timepars['slope'].value
self.intercept = timepars['intercept']
except:
pass
def test_array_display():
from ctapipe.visualization.mpl_array import ArrayDisplay
from ctapipe.image.timing_parameters import timing_parameters
# build a test subarray:
tels = dict()
tel_pos = dict()
for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m):
tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam")
tel_pos[ii + 1] = pos
sub = SubarrayDescription(name="TestSubarray",
tel_positions=tel_pos,
tel_descriptions=tels)
ad = ArrayDisplay(sub)
ad.set_vector_rho_phi(1 * u.m, 90 * u.deg)
# try setting a value
vals = ones(sub.num_tels)
ad.values = vals
assert (vals == ad.values).all()
# test using hillas params:
hillas_dict = {
1: HillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg),
2: HillasParametersContainer(length=20000 * u.cm, psi="95deg"),
}
grad = 2
intercept = 1
rot_angle = 20 * u.deg
timing_rot20 = timing_parameters(pix_x=arange(4) * u.deg,
pix_y=zeros(4) * u.deg,
image=ones(4),
peak_time=intercept * u.ns +
grad * arange(4) * u.ns,
rotation_angle=rot_angle)
gradient_dict = {
1: timing_rot20.gradient.value,
2: timing_rot20.gradient.value,
}
ad.set_vector_hillas(hillas_dict=hillas_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg)
ad.set_line_hillas(hillas_dict, range=300)
ad.add_labels()
ad.remove_labels()
def test_ignore_negative():
grad = 2.0
intercept = 1.0
geom = CameraGeometry.from_name("LSTCam")
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg)
image = np.ones(geom.n_pixels)
image[5:10] = -1.0
timing = timing_parameters(
geom,
image,
peakpos=intercept + grad * geom.pix_x.value,
hillas_parameters=hillas,
)
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit)
assert_allclose(timing.intercept, intercept)
def test_psi_20():
# Then try a different rotation angle
grad = 2
intercept = 1
geom = CameraGeometry.from_name("LSTCam")
psi = 20 * u.deg
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=psi)
timing = timing_parameters(
geom,
image=np.ones(geom.n_pixels),
peakpos=intercept + grad * (np.cos(psi) * geom.pix_x.value
+ np.sin(psi) * geom.pix_y.value),
hillas_parameters=hillas,
)
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit)
assert_allclose(timing.intercept, intercept)
def test_psi_0():
"""
Simple test that gradient fitting gives expected answers for perfect
gradient
"""
grad = 2.0
intercept = 1.0
geom = CameraGeometry.from_name("LSTCam")
hillas = HillasParametersContainer(x=0 * u.m, y=0 * u.m, psi=0 * u.deg)
timing = timing_parameters(
geom,
image=np.ones(geom.n_pixels),
peakpos=intercept + grad * geom.pix_x.value,
hillas_parameters=hillas,
)
# Test we get the values we put in back out again
assert_allclose(timing.slope, grad / geom.pix_x.unit)
assert_allclose(timing.intercept, intercept)
def calculate_image_features(telescope_id, event, dl1, config):
''' Performs cleaning and adds the following image parameters:
- hillas
- leakage
- concentration
- timing
- number of islands
Make sure to adapt cleaning levels to the used algorithm (-> config)
- tailcuts:
picture_thresh, picture_thresh, min_number_picture_neighbors
- fact_image_cleaning:
picture_threshold, boundary_threshold, min_number_neighbors, time_limit
Returns:
--------
TelescopeParameterContainer
'''
array_event_id = event.dl0.event_id
run_id = event.r0.obs_id
telescope = event.inst.subarray.tels[telescope_id]
image = dl1.image
# might want to make the parameter names more consistent between methods
if config.cleaning_method == 'tailcuts_clean':
boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[
telescope.camera.cam_id
]
mask = tailcuts_clean(
telescope.camera,
image,
boundary_thresh=boundary_thresh,
picture_thresh=picture_thresh,
min_number_picture_neighbors=min_number_picture_neighbors,
)
elif config.cleaning_method == 'fact_image_cleaning':
boundary_threshold, picture_threshold, time_limit, min_number_neighbors = config.cleaning_level[
telescope.camera.cam_id
]
mask = fact_image_cleaning(
telescope.camera,
image,
dl1.pulse_time,
boundary_threshhold=boundary_threshold,
picture_threshold=picture_threshold,
min_number_neighbors=min_number_neighbors,
time_limit=time_limit,
)
cleaned = image.copy()
cleaned[~mask] = 0
logging.debug(f'calculating hillas for event {array_event_id, telescope_id}')
hillas_container = hillas_parameters(telescope.camera, cleaned)
hillas_container.prefix = ''
logging.debug(f'calculating leakage for event {array_event_id, telescope_id}')
leakage_container = leakage(telescope.camera, image, mask)
leakage_container.prefix = ''
logging.debug(f'calculating concentration for event {array_event_id, telescope_id}')
concentration_container = concentration(telescope.camera, image, hillas_container)
concentration_container.prefix = ''
logging.debug(f'getting timing information for event {array_event_id, telescope_id}')
timing_container = timing_parameters(telescope.camera, image, dl1.pulse_time, hillas_container)
timing_container.prefix = ''
# membership missing for now as it causes problems with the hdf5tablewriter
# right now i dont need this anyway
logging.debug(f'calculating num_islands for event {array_event_id, telescope_id}')
num_islands, membership = number_of_islands(telescope.camera, mask)
island_container = IslandContainer(num_islands=num_islands)
island_container.prefix = ''
num_pixel_in_shower = mask.sum()
logging.debug(f'getting pointing container for event {array_event_id, telescope_id}')
# ctapipe requires this to be rad
pointing_container = TelescopePointingContainer(
azimuth=event.mc.tel[telescope_id].azimuth_raw * u.rad,
altitude=event.mc.tel[telescope_id].altitude_raw * u.rad,
prefix='pointing',
)
return TelescopeParameterContainer(
telescope_id=telescope_id,
run_id=run_id,
array_event_id=array_event_id,
leakage=leakage_container,
hillas=hillas_container,
concentration=concentration_container,
pointing=pointing_container,
timing=timing_container,
islands=island_container,
telescope_type_id=config.types_to_id[telescope.type],
camera_type_id=config.names_to_id[telescope.camera.cam_id],
focal_length=telescope.optics.equivalent_focal_length,
mirror_area=telescope.optics.mirror_area,
num_pixel_in_shower=num_pixel_in_shower,
)
continue
# Calculate image parameters
hillas = hillas_parameters(
geom,
clean) # this one gives some warnings invalid value in sqrt
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean,
peak_time, hillas.psi)
if w >= 0:
if fitsdata.size == 0:
fitsdata = clean
else:
fitsdata = np.vstack([fitsdata, clean]) #Pixel content
camtype.append(str(geom))
width = np.append(width, w.value)
length = np.append(length, l.value)
phi = np.append(phi, hillas.phi)
psi = np.append(psi, hillas.psi)
r = np.append(r, hillas.r)
x = np.append(x, hillas.x)
y = np.append(y, hillas.y)
def get_events(filename,
storedata=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Read a Simtelarray file, extract pixels charge, calculate image parameters and timing
parameters and store the result in an hdf5 file.
Parameters:
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
"""
#Particle type:
particle_type = guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'ObsID', 'EvID', 'mcEnergy', 'mcAlt', 'mcAz', 'mcCore_x', 'mcCore_y',
'mcHfirst', 'mcType', 'GPStime', 'width', 'length', 'w/l', 'phi',
'psi', 'r', 'x', 'y', 'intensity', 'skewness', 'kurtosis', 'mcAlttel',
'mcAztel', 'impact', 'mcXmax', 'time_gradient', 'intercept', 'SrcX',
'SrcY', 'disp', 'hadroness'
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = EventSourceFactory.produce(
input_url=filename, allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
'''
if i > 100: # for quick tests
break
'''
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal
# the pedestal is the average (for pedestal events) of the *sum* of all samples, from sim_telarray
nsamples = data.shape[2] # total number of samples
pedcorrectedsamples = data - np.atleast_3d(
ped
) / nsamples # Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
integrator = LocalPeakIntegrator(None, None)
integration, peakpos, window = integrator.extract_charge(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(
np.log10(signals)
) # This seems to be faster like this, with normal python lists
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(
cleanmask
) # This seems to be faster like this, with normal python lists
clean = signals.copy()
clean[
~cleanmask] = 0.0 # set to 0 pixels which did not survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(
geom,
clean) # this one gives some warnings invalid value in sqrt
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(peakpos[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom.pix_x, geom.pix_y, clean,
peak_time, hillas.psi)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#Store parameters from event and MC:
ObsID = event.r0.obs_id
EvID = event.r0.event_id
mcEnergy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mcAlt = event.mc.alt.value
mcAz = event.mc.az.value
mcCore_x = event.mc.core_x.value
mcCore_y = event.mc.core_y.value
mcHfirst = event.mc.h_first_int.value
mcType = event.mc.shower_primary_id
mcAztel = event.mcheader.run_array_direction[0].value
mcAlttel = event.mcheader.run_array_direction[1].value
mcXmax = event.mc.x_max.value
GPStime = event.trig.gps_time.value
impact = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
time_gradient = timepars[0].value
intercept = timepars[1].value
#Calculate Disp and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = transformations.calc_CamSourcePos(
mcAlt, mcAz, mcAlttel, mcAztel, focal_length)
SrcX = sourcepos[0]
SrcY = sourcepos[1]
disp = transformations.calc_DISP(sourcepos[0], sourcepos[1], x,
y)
hadroness = 0
if particle_type == 'proton':
hadroness = 1
eventdf = pd.DataFrame([[
ObsID, EvID, mcEnergy, mcAlt, mcAz, mcCore_x, mcCore_y,
mcHfirst, mcType, GPStime, width, length, width / length,
phi, psi, r, x, y, intensity, skewness, kurtosis, mcAlttel,
mcAztel, impact, mcXmax, time_gradient, intercept, SrcX,
SrcY, disp, hadroness
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if concatenate == False or (concatenate == True and
np.DataSource().exists(outfile) == False):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
def process_event(event, config):
'''
Processes
'''
reco_algorithm = config.reco_algorithm
features = {}
params = {}
pointing_azimuth = {}
pointing_altitude = {}
tel_x = {}
tel_y = {}
tel_focal_lengths = {}
cleaning_method = config.cleaning_method
valid_cleaning_methods = ['tailcuts_clean', 'fact_image_cleaning']
if cleaning_method not in valid_cleaning_methods:
print('Cleaning Method not implemented')
print('Please use one of ', valid_cleaning_methods)
return None
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
telescope_type_name = event.inst.subarray.tels[
telescope_id].optics.tel_type
if cleaning_method == 'tailcuts_clean':
boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[
camera.cam_id]
mask = tailcuts_clean(camera, dl1.image[0],
*config.cleaning_level[camera.cam_id])
elif cleaning_method == 'fact_image_cleaning':
mask = fact_image_cleaning(
camera, dl1.image[0],
*config.cleaning_level_fact[camera.cam_id])
try:
cleaned = dl1.image[0].copy()
cleaned[~mask] = 0
hillas_container = hillas_parameters(
camera,
cleaned,
)
params[telescope_id] = hillas_container
except HillasParameterizationError:
continue
# probably wise to add try...except blocks here as well
# Add more Features here (look what ctapipe can do, timing?)
num_islands, island_labels = number_of_islands(camera, mask)
island_dict = {
'num_islands': num_islands,
'island_labels': island_labels
}
leakage_container = leakage(camera, dl1.image[0], mask)
timing_container = timing_parameters(camera, dl1.image[0],
dl1.peakpos[0], hillas_container)
pointing_azimuth[
telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[
telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0]
tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1]
telescope_description = event.inst.subarray.tel[telescope_id]
tel_focal_lengths[
telescope_id] = telescope_description.optics.equivalent_focal_length
d = {
'array_event_id': event.dl0.event_id,
'telescope_id': int(telescope_id),
'camera_name': camera.cam_id,
'camera_id': config.names_to_id[camera.cam_id],
'run_id': event.r0.obs_id,
'telescope_type_name': telescope_type_name,
'telescope_type_id': config.types_to_id[telescope_type_name],
'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw,
'pointing_altitude': event.mc.tel[telescope_id].altitude_raw,
'mirror_area': telescope_description.optics.mirror_area,
'focal_length':
telescope_description.optics.equivalent_focal_length,
}
d.update(hillas_container.as_dict())
d.update(leakage_container.as_dict())
d.update(island_dict)
d.update(timing_container.as_dict())
features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()})
if reco_algorithm == 'intersection':
reco = HillasIntersection()
array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1],
az=event.mcheader.run_array_direction[0],
frame='altaz')
reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths,
array_direction)
elif reco_algorithm == 'planes':
reco = HillasReconstructor()
reconstruction = reco.predict(params, event.inst, pointing_altitude,
pointing_azimuth)
for telescope_id in event.dl1.tel.keys():
if telescope_id not in params:
continue
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
pos = event.inst.subarray.positions[telescope_id]
x, y = pos[0], pos[1]
core_x = reconstruction.core_x
core_y = reconstruction.core_y
d = np.sqrt((core_x - x)**2 + (core_y - y)**2)
features[telescope_id]['distance_to_core'] = d.value
return pd.DataFrame(list(features.values())), reconstruction, params
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(geom[clean], image[clean])
leakage_c = leakage(geom, image, clean)
n_islands, island_ids = number_of_islands(geom, clean)
timing_c = timing_parameters(
geom[clean],
image[clean],
peakpos[clean],
hillas_c,
)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if geom.camera_name != 'ASTRICam':
time_gradients[telescope_id] = timing_c.slope.value
else:
time_gradients[telescope_id] = hillas_c.skewness
print(geom.camera_name, time_gradients[telescope_id])
# make sure each telescope get's an arrow
def set_timing_features(self, geom, image, pulse_time, hillas):
peak_time = Quantity(pulse_time) * u.Unit("ns")
timepars = time.timing_parameters(geom, image, peak_time, hillas)
self.time_gradient = timepars.slope.value
self.intercept = timepars.intercept
def get_events(filename,
storedata=False,
test=False,
concatenate=False,
storeimg=False,
outdir='./results/'):
"""
Depreciated, use r0_to_dl1.
Read a Simtelarray file, extract pixels charge, calculate image
parameters and timing parameters and store the result in an hdf5
file.
Parameters:
-----------
filename: str
Name of the simtelarray file.
storedata: boolean
True: store extracted data in a hdf5 file
concatenate: boolean
True: store the extracted data at the end of an existing file
storeimg: boolean
True: store also pixel data
outdir: srt
Output directory
Returns:
--------
pandas DataFrame: output
"""
from warnings import warn
warn("Deprecated: use r0_to_dl1")
#Particle type:
particle_type = utils.guess_type(filename)
#Create data frame where DL2 data will be stored:
features = [
'obs_id',
'event_id',
'mc_energy',
'mc_alt',
'mc_az',
'mc_core_x',
'mc_core_y',
'mc_h_first_int',
'mc_type',
'gps_time',
'width',
'length',
'wl',
'phi',
'psi',
'r',
'x',
'y',
'intensity',
'skewness',
'kurtosis',
'mc_alt_tel',
'mc_az_tel',
'mc_core_distance',
'mc_x_max',
'time_gradient',
'intercept',
'src_x',
'src_y',
'disp_norm',
]
output = pd.DataFrame(columns=features)
#Read LST1 events:
source = event_source(input_url=filename,
allowed_tels={1}) #Open Simtelarray file
#Cleaning levels:
level1 = {'LSTCam': 6.}
level2 = level1.copy()
# We use as second cleaning level just half of the first cleaning level
for key in level2:
level2[key] *= 0.5
log10pixelHGsignal = {}
survived = {}
imagedata = np.array([])
for key in level1:
log10pixelHGsignal[key] = []
survived[key] = []
i = 0
for event in source:
if i % 100 == 0:
print("EVENT_ID: ", event.r0.event_id, "TELS: ",
event.r0.tels_with_data, "MC Energy:", event.mc.energy)
i = i + 1
ntels = len(event.r0.tels_with_data)
if test == True and i > 1000: # for quick tests
break
for ii, tel_id in enumerate(event.r0.tels_with_data):
geom = event.inst.subarray.tel[tel_id].camera #Camera geometry
tel_coords = event.inst.subarray.tel_coords[
event.inst.subarray.tel_indices[tel_id]]
data = event.r0.tel[tel_id].waveform
ped = event.mc.tel[tel_id].pedestal # the pedestal is the
#average (for pedestal events) of the *sum* of all samples,
#from sim_telarray
nsamples = data.shape[2] # total number of samples
# Subtract pedestal baseline. atleast_3d converts 2D to 3D matrix
pedcorrectedsamples = data - np.atleast_3d(ped) / nsamples
integrator = LocalPeakWindowSum()
integration, pulse_time = integrator(
pedcorrectedsamples
) # these are 2D matrices num_gains * num_pixels
chan = 0 # high gain used for now...
signals = integration[chan].astype(float)
dc2pe = event.mc.tel[tel_id].dc_to_pe # numgains * numpixels
signals *= dc2pe[chan]
# Add all individual pixel signals to the numpy array of the
# corresponding camera inside the log10pixelsignal dictionary
log10pixelHGsignal[str(geom)].extend(np.log10(signals))
# Apply image cleaning
cleanmask = tailcuts_clean(geom,
signals,
picture_thresh=level1[str(geom)],
boundary_thresh=level2[str(geom)],
keep_isolated_pixels=False,
min_number_picture_neighbors=1)
survived[str(geom)].extend(cleanmask)
clean = signals.copy()
clean[~cleanmask] = 0.0 # set to 0 pixels which did not
# survive cleaning
if np.max(clean) < 1.e-6: # skip images with no pixels
continue
# Calculate image parameters
hillas = hillas_parameters(geom, clean)
foclen = event.inst.subarray.tel[
tel_id].optics.equivalent_focal_length
w = np.rad2deg(np.arctan2(hillas.width, foclen))
l = np.rad2deg(np.arctan2(hillas.length, foclen))
#Calculate Timing parameters
peak_time = units.Quantity(pulse_time[chan]) * units.Unit("ns")
timepars = time.timing_parameters(geom, clean, peak_time, hillas)
if w >= 0:
if storeimg == True:
if imagedata.size == 0:
imagedata = clean
else:
imagedata = np.vstack([imagedata,
clean]) #Pixel content
#Hillas parameters
width = w.value
length = l.value
phi = hillas.phi.value
psi = hillas.psi.value
r = hillas.r.value
x = hillas.x.value
y = hillas.y.value
intensity = np.log10(hillas.intensity)
skewness = hillas.skewness
kurtosis = hillas.kurtosis
#MC data:
obs_id = event.r0.obs_id
event_id = event.r0.event_id
mc_energy = np.log10(event.mc.energy.value *
1e3) #Log10(Energy) in GeV
mc_alt = event.mc.alt.value
mc_az = event.mc.az.value
mc_core_x = event.mc.core_x.value
mc_core_y = event.mc.core_y.value
mc_h_first_int = event.mc.h_first_int.value
mc_type = event.mc.shower_primary_id
mc_az_tel = event.mcheader.run_array_direction[0].value
mc_alt_tel = event.mcheader.run_array_direction[1].value
mc_x_max = event.mc.x_max.value
gps_time = event.trig.gps_time.value
#Calculate mc_core_distance parameters
mc_core_distance = np.sqrt(
(tel_coords.x.value - event.mc.core_x.value)**2 +
(tel_coords.y.value - event.mc.core_y.value)**2)
#Timing parameters
time_gradient = timepars['slope'].value
intercept = timepars['intercept']
#Calculate disp_ and Source position in camera coordinates
tel = OpticsDescription.from_name(
'LST') #Telescope description
focal_length = tel.equivalent_focal_length.value
sourcepos = utils.cal_cam_source_pos(mc_alt, mc_az, mc_alt_tel,
mc_az_tel, focal_length)
src_x = sourcepos[0]
src_y = sourcepos[1]
disp = utils.disp_norm(sourcepos[0], sourcepos[1], x, y)
eventdf = pd.DataFrame([[
obs_id, event_id, mc_energy, mc_alt, mc_az, mc_core_x,
mc_core_y, mc_h_first_int, mc_type, gps_time, width,
length, width / length, phi, psi, r, x, y, intensity,
skewness, kurtosis, mc_alt_tel, mc_az_tel,
mc_core_distance, mc_x_max, time_gradient, intercept,
src_x, src_y, disp, mc_type
]],
columns=features)
output = output.append(eventdf, ignore_index=True)
outfile = outdir + particle_type + '_events.hdf5'
if storedata == True:
if (concatenate == False
or (concatenate == True
and np.DataSource().exists(outfile) == False)):
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
if storeimg == True:
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=imagedata)
f.close()
else:
if storeimg == True:
f = h5py.File(outfile, 'r')
images = f['images']
del f['images']
images = np.vstack([images, imagedata])
f.close()
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
f = h5py.File(outfile, 'r+')
f.create_dataset('images', data=images)
f.close()
else:
saved = pd.read_hdf(outfile, key=particle_type + '_events')
output = saved.append(output, ignore_index=True)
output.to_hdf(outfile, key=particle_type + "_events", mode="w")
del source
return output
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors
)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(camera[clean], image[clean])
leakage_c = leakage(camera, image, clean)
n_islands, island_ids = number_of_islands(camera, clean)
timing_c = timing_parameters(
camera[clean], image[clean], peakpos[clean], hillas_c
)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
pointing_azimuth[telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if camera.cam_id != 'ASTRICam':
time_gradients[telescope_id] = timing_c.slope.value
else:
time_gradients[telescope_id] = hillas_c.skewness
print(camera.cam_id, time_gradients[telescope_id])
def set_timing_features(self, geom, image, pulse_time, hillas):
timepars = time.timing_parameters(geom, image, pulse_time, hillas)
self.time_gradient = timepars.slope.value
self.intercept = timepars.intercept
