def test_camera_calibrator(example_event):
telid = list(example_event.r0.tel)[0]
calibrator = CameraCalibrator()
calibrator.calibrate(example_event)
image = example_event.dl1.tel[telid].image
assert image is not None
def test_eventsource_override_r1():
dataset = get_dataset("gamma_test.simtel.gz")
eventsource = HESSIOEventSource(input_url=dataset)
calibrator = CameraCalibrator(
eventsource=eventsource,
r1_product="NullR1Calibrator"
)
assert isinstance(calibrator.r1, NullR1Calibrator)
def test_camera_calibrator(example_event):
telid = 11
calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator")
calibrator.calibrate(example_event)
image = example_event.dl1.tel[telid].image
assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator(test_event):
event = deepcopy(test_event) # so we don't modify the test event
telid = 11
calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator")
calibrator.calibrate(event)
image = event.dl1.tel[telid].image
assert_allclose(image[0, 0], -2.216, 1e-3)
def test_camera_calibrator():
event = get_test_event()
telid = 11
calibrator = CameraCalibrator(None, None)
calibrator.calibrate(event)
image = event.dl1.tel[telid].image
assert_allclose(image[0, 0], -2.216, 1e-3)
def __init__(self,
event,
telescope_list,
camera_types,
ChargeExtration,
pe_thresh,
min_neighbors,
tail_thresholds,
DirReco,
quality_cuts,
LUT=None):
super().__init__()
'''
Parmeters
---------
event : ctapipe event container
calibrator : ctapipe camera calibrator
reconstructor : ctapipe hillas reconstructor
telescope_list : list with telescope configuration or "all"
pe_thresh : dict with thresholds for gain selection
tail_thresholds : dict with thresholds for image cleaning
quality_cuts : dict containing quality cuts
canera_types : list with camera types to analyze
'''
self.event = event
self.telescope_list = telescope_list
self.pe_thresh = pe_thresh
self.min_neighbors = min_neighbors
self.tail_thresholds = tail_thresholds
self.quality_cuts = quality_cuts
self.camera_types = camera_types
self.dirreco = DirReco
self.LUTgenerator = LUT
if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"]
== "doublepass"):
self.weights = {}
else:
self.weights = None
self.hillas_dict = {}
self.camera_dict = {}
self.edge_pixels = {}
# configurations for calibrator
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = \
ChargeExtration["ChargeExtractorProduct"]
cfg['WaveformCleanerFactory']['product'] = \
ChargeExtration["WaveformCleanerProduct"]
self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator",
config=cfg) # calibration
self.reconstructor = HillasReconstructor() # direction
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
kwargs = dict(config=self.config, tool=self)
reader_factory = EventFileReaderFactory(**kwargs)
reader_class = reader_factory.get_class()
self.reader = reader_class(**kwargs)
self.calibrator = CameraCalibrator(origin=self.reader.origin, **kwargs)
self.plotter = ImagePlotter(**kwargs)
def test_config():
window_shift = 3
window_width = 9
config = Config({"LocalPeakWindowSum": {
"window_shift": window_shift,
"window_width": window_width,
}})
calibrator = CameraCalibrator(
extractor_name='LocalPeakWindowSum',
config=config
)
assert calibrator.dl1.extractor.window_shift == window_shift
assert calibrator.dl1.extractor.window_width == window_width
def test_eventsource_r1():
dataset = get_dataset_path("gamma_test.simtel.gz")
eventsource = HESSIOEventSource(input_url=dataset)
calibrator = CameraCalibrator(eventsource=eventsource)
assert isinstance(calibrator.r1, HESSIOR1Calibrator)
def test_manual_extractor():
calibrator = CameraCalibrator(extractor_product="LocalPeakIntegrator")
assert isinstance(calibrator.dl1.extractor, LocalPeakIntegrator)
def test_manual_r1():
calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator")
assert isinstance(calibrator.r1, HESSIOR1Calibrator)
def r0_to_dl1(
input_filename=get_dataset_path('gamma_test_large.simtel.gz'),
output_filename=None,
custom_config={},
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
Returns
-------
"""
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
custom_calibration = config["custom_calibration"]
source = EventSource(input_url=input_filename,
config=Config(config["source_config"]))
subarray = source.subarray
is_simu = source.is_simulation
metadata = global_metadata(source)
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = CameraCalibrator(
image_extractor_type=config['image_extractor'],
config=Config(config),
subarray=subarray)
if not is_simu:
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
image_extractor_type=config['image_extractor_for_muons'],
config=Config(config),
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config(config[config['calibration_product']])
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# Write extra information to the DL1 file
write_array_info(subarray, output_filename)
write_array_info_08(subarray, output_filename)
if is_simu:
write_mcheader(
source.simulation_config,
output_filename,
obs_id=source.obs_ids[0],
filters=HDF5_ZSTD_FILTERS,
metadata=metadata,
)
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=HDF5_ZSTD_FILTERS,
add_prefix=True,
# overwrite=True,
) as writer:
if is_simu:
subarray = subarray
# build a mapping of tel_id back to tel_index:
# (note this should be part of SubarrayDescription)
idx = np.zeros(max(subarray.tel_indices) + 1)
for key, val in subarray.tel_indices.items():
idx[key] = val
# the final transform then needs the mapping and the number of telescopes
tel_list_transform = partial(
utils.expand_tel_list,
max_tels=max(subarray.tel) + 1,
)
writer.add_column_transform(table_name='subarray/trigger',
col_name='tels_with_trigger',
transform=tel_list_transform)
# Forcing filters for the dl1 dataset that are currently read from the pre-existing files
# This should be fixed in ctapipe and then corrected here
writer._h5file.filters = HDF5_ZSTD_FILTERS
logger.info(f"USING FILTERS: {writer._h5file.filters}")
for i, event in enumerate(source):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.trigger.prefix = ''
if event.simulation is not None:
event.simulation.prefix = 'mc'
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
if not custom_calibration:
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
#initialize the event monitoring data
event.mon = source.r0_r1_calibrator.mon_data
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=True,
new_ff=True)
# flat-field or pedestal:
if (event.trigger.event_type == EventType.FLATFIELD
or event.trigger.event_type == EventType.SKY_PEDESTAL):
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and gain select the event by hand for DL1
source.r0_r1_calibrator.calibrate(event)
# create image for all events
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for ii, telescope_id in enumerate(event.dl1.tel.keys()):
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
tel = event.dl1.tel[telescope_id]
tel.prefix = '' # don't really need one
# remove the first part of the tel_name which is the type 'LST', 'MST' or 'SST'
tel_name = str(subarray.tel[telescope_id])[4:]
if custom_calibration:
lst_calibration(event, telescope_id)
write_event = True
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
assert event.dl1.tel[telescope_id].image is not None
try:
get_dl1(
event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
dl1_container.ucts_time = 0
# convert Time to unix timestamp in (UTC) to keep compatibility
# with older lstchain
# FIXME: just keep it as time, table writer and reader handle it
dl1_container.dragon_time = event.trigger.time.unix
dl1_container.tib_time = 0
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
dl1_container.trigger_type = event.lst.tel[
telescope_id].evt.tib_masked_trigger
else:
dl1_container.trigger_type = event.trigger.event_type
dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
dl1_container.alt_tel = event.pointing.tel[
telescope_id].altitude
dl1_container.trigger_time = event.trigger.time.unix
dl1_container.event_type = event.trigger.event_type
# FIXME: no need to read telescope characteristics like foclen for every event!
foclen = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = u.Quantity(
subarray.tel[telescope_id].optics.mirror_area, u.m**2)
dl1_container.prefix = tel.prefix
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
for container in [extra_im, dl1_container, event.r0, tel]:
add_global_metadata(container, metadata)
event.r0.prefix = ''
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, tel, extra_im])
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
1] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_pixels = bad_pixels_hg | bad_pixels_lg
bad_waveform = np.transpose(
np.array(numsamples * [bad_pixels]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
tel = event.dl1.tel[telescope_id]
image = tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, foclen,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
bright_pixels = image > min_pe_for_muon_t_calc
selected_gain = event.r1.tel[
telescope_id].selected_gain_channel
mask_hg = bright_pixels & (selected_gain == 0)
mask_lg = bright_pixels & (selected_gain == 1)
bright_pixels_waveforms_hg = event.r1.tel[
telescope_id].waveform[mask_hg, :]
bright_pixels_waveforms_lg = event.r1.tel[
telescope_id].waveform[mask_lg, :]
stacked_waveforms_hg = np.sum(
bright_pixels_waveforms_hg, axis=0)
stacked_waveforms_lg = np.sum(
bright_pixels_waveforms_lg, axis=0)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms_hg,
axis=-1)
lg_peak_sample = np.argmax(stacked_waveforms_lg,
axis=-1)
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id,
dl1_container.dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if (is_simu and config['write_pe_image']
and event.simulation.tel[telescope_id].true_image
is not None and
event.simulation.tel[telescope_id].true_image.any()):
event.simulation.tel[telescope_id].prefix = ''
writer.write(table_name=f'simulation/{tel_name}',
containers=[
event.simulation.tel[telescope_id],
extra_im
])
if not is_simu:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if is_simu:
# Reconstruct source position from disp for all events and write the result in the output file
for tel_name in ['LST_LSTCam']:
focal = OpticsDescription.from_name(
tel_name.split('_')[0]).equivalent_focal_length
add_disp_to_parameters_table(output_filename,
dl1_params_lstcam_key, focal)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
else:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def calculate_required_additional_nsb(simtel_filename,
data_dl1_filename,
config=None):
# TODO check if good estimation
# TODO reduce duplicated code with 'calculate_noise_parameters'
"""
Calculates the additional NSB needed in the MC waveforms
to match a real data DL1 file
Parameters
----------
simtel_filename: a simtel file containing showers, from the production
(same NSB and telescope settings) as the one on which the correction will
be applied. It must contain pixel-wise info on true number of p.e.'s from
C-photons (will be used to identify pixels which only contain noise).
data_dl1_filename: a real data DL1 file (processed with calibration
settings corresponding to those with which the MC is to be processed).
It must contain calibrated images, i.e. "DL1a" data. This file has the
"target" NSB which we want to have in the MC files, for better
agreement of data and simulations.
config: configuration containing the calibration
settings used for processing both the data and the MC files above
Returns
-------
extra_nsb: Fraction of the additional NSB in data compared to MC.
data_ped_variance: Pedestal variance from data
mc_ped_variance: Pedestal variance from MC
"""
log.setLevel(logging.INFO)
if config is None:
config = standard_config
# Real data DL1 tables:
data_dl1_calibration = read_table(
data_dl1_filename, '/dl1/event/telescope/monitoring/calibration')
data_dl1_pedestal = read_table(data_dl1_filename,
'/dl1/event/telescope/monitoring/pedestal')
unusable = data_dl1_calibration['unusable_pixels']
# Locate pixels with HG declared unusable either in original calibration or
# in interleaved events:
bad_pixels = unusable[0][0] # original calibration
for tf in unusable[1:][0]: # calibrations with interleaved
bad_pixels = np.logical_or(bad_pixels, tf)
good_pixels = ~bad_pixels
# First index: 1,2,... = values from interleaved (0 is for original
# calibration run)
# Second index: 0 = high gain
# Third index: pixels
# HG adc to pe conversion factors from interleaved calibrations:
data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :]
# Pixel-wise pedestal standard deviation (for an unbiased extractor),
# in adc counts:
data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :]
# indices which connect each pedestal calculation to a given calibration:
calibration_id = data_dl1_pedestal['calibration_id'][1:]
# convert pedestal st deviations to p.e.
dummy = []
for i, x in enumerate(data_HG_ped_std[:, ]):
dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ])
dummy = np.array(dummy)
# Average for all interleaved calibrations (in case there are more than one)
data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel
# Identify noisy pixels, likely containing stars - we want to adjust MC to
# the average diffuse NSB across the camera
data_median_std_ped_pe = np.median(data_HG_ped_std_pe)
data_std_std_ped_pe = np.std(data_HG_ped_std_pe)
log.info(f'Real data: median across camera of good pixels\' pedestal std '
f'{data_median_std_ped_pe:.3f} p.e.')
brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe
too_bright_pixels = (data_HG_ped_std_pe > brightness_limit)
log.info(f'Number of pixels beyond 3 std dev of median: '
f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)')
# Exclude too bright pixels, besides those with unusable calibration:
good_pixels &= ~too_bright_pixels
# recalculate the median of the pixels' std dev, with good_pixels:
data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels])
log.info(f'Good and not too bright pixels: {good_pixels.sum()}')
# Event reader for simtel file:
mc_reader = EventSource(input_url=simtel_filename, config=Config(config))
# Obtain the configuration with which the pedestal calculations were
# performed:
ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator']
tel_id = ped_config['tel_id']
# Obtain the (unbiased) extractor used for pedestal calculations:
pedestal_calibrator = CameraCalibrator(
image_extractor_type=ped_config['charge_product'],
config=Config(config['LSTCalibrationCalculator']),
subarray=mc_reader.subarray)
# Since these extractors are now for use on MC, we have to apply the pulse
# integration correction (in data that is currently, as of
# lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the
# adc to pe conversion factors )
pedestal_calibrator.image_extractors[
ped_config['charge_product']].apply_integration_correction = True
# MC pedestals integrated with the unbiased pedestal extractor
mc_ped_charges = []
for event in mc_reader:
if tel_id not in event.trigger.tels_with_trigger:
continue
# Extract the signals as we do for pedestals (unbiased fixed window
# extractor):
pedestal_calibrator(event)
charges = event.dl1.tel[tel_id].image
# True number of pe's from Cherenkov photons (to identify noise-only pixels)
true_image = event.simulation.tel[tel_id].true_image
mc_ped_charges.append(charges[true_image == 0])
# All pixels behave (for now) in the same way in MC, just put them together
mc_ped_charges = np.concatenate(mc_ped_charges)
mc_unbiased_std_ped_pe = np.std(mc_ped_charges)
# Find the additional noise (in data w.r.t. MC) for the unbiased extractor
# The idea is that pedestal variance scales with NSB
data_ped_variance = data_median_std_ped_pe**2
mc_ped_variance = mc_unbiased_std_ped_pe**2
extra_nsb = ((data_ped_variance - mc_ped_variance) / mc_ped_variance)
return extra_nsb, data_ped_variance, mc_ped_variance
def calculate_noise_parameters(simtel_filename,
data_dl1_filename,
config_filename=None):
"""
Calculates the parameters needed to increase the noise in an MC DL1 file
to match the noise in a real data DL1 file, using add_noise_in_pixels
The returned parameters are those needed by the function add_noise_in_pixels (see
description in its documentation above).
Parameters
----------
simtel_filename: `str`
a simtel file containing showers, from the same
production (same NSB and telescope settings) as the MC DL1 file below. It
must contain pixel-wise info on true number of p.e.'s from C-photons (
will be used to identify pixels which only contain noise).
data_dl1_filename: `str`
a real data DL1 file (processed with calibration
settings corresponding to those with which the MC is to be processed).
It must contain calibrated images, i.e. "DL1a" data. This file has the
"target" noise which we want to have in the MC files, for better
agreement of data and simulations.
config_filename: `str`
configuration file containing the calibration
settings used for processing both the data and the MC files above
Returns
-------
extra_noise_in_dim_pixels: `float`
Extra noise of dim pixels.
extra_bias_in_dim_pixels: `float`
Extra bias of dim pixels.
extra_noise_in_bright_pixels: `float`
Extra noise of bright pixels
"""
log.setLevel(logging.INFO)
if config_filename is None:
config = standard_config
else:
config = read_configuration_file(config_filename)
# Real data DL1 tables:
data_dl1_calibration = read_table(
data_dl1_filename, '/dl1/event/telescope/monitoring/calibration')
data_dl1_pedestal = read_table(data_dl1_filename,
'/dl1/event/telescope/monitoring/pedestal')
data_dl1_parameters = read_table(
data_dl1_filename, '/dl1/event/telescope/parameters/LST_LSTCam')
data_dl1_image = read_table(data_dl1_filename,
'/dl1/event/telescope/image/LST_LSTCam')
unusable = data_dl1_calibration['unusable_pixels']
# Locate pixels with HG declared unusable either in original calibration or
# in interleaved events:
bad_pixels = unusable[0][0] # original calibration
for tf in unusable[1:][0]: # calibrations with interleaveds
bad_pixels = np.logical_or(bad_pixels, tf)
good_pixels = ~bad_pixels
# First index: 1,2,... = values from interleaveds (0 is for original
# calibration run)
# Second index: 0 = high gain
# Third index: pixels
# HG adc to pe conversion factors from interleaved calibrations:
data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :]
# Pixel-wise pedestal standard deviation (for an unbiased extractor),
# in adc counts:
data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :]
# indices which connect each pedestal calculation to a given calibration:
calibration_id = data_dl1_pedestal['calibration_id'][1:]
# convert pedestal st deviations to p.e.
dummy = []
for i, x in enumerate(data_HG_ped_std[:, ]):
dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ])
dummy = np.array(dummy)
# Average for all interleaved calibrations (in case there are more than one)
data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel
# Identify noisy pixels, likely containing stars - we want to adjust MC to
# the average diffuse NSB across the camera
data_median_std_ped_pe = np.median(data_HG_ped_std_pe)
data_std_std_ped_pe = np.std(data_HG_ped_std_pe)
log.info(f'Real data: median across camera of good pixels\' pedestal std '
f'{data_median_std_ped_pe:.3f} p.e.')
brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe
too_bright_pixels = (data_HG_ped_std_pe > brightness_limit)
log.info(f'Number of pixels beyond 3 std dev of median: '
f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)')
ped_mask = data_dl1_parameters['event_type'] == 2
# The charges in the images below are obtained with the extractor for
# showers, usually a biased one, like e.g. LocalPeakWindowSum
data_ped_charges = data_dl1_image['image'][ped_mask]
# Exclude too bright pixels, besides those with unusable calibration:
good_pixels &= ~too_bright_pixels
# recalculate the median of the pixels' std dev, with good_pixels:
data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels])
log.info(f'Good and not too bright pixels: {good_pixels.sum()}')
# all_good is an events*pixels boolean array of valid signals:
all_good = np.reshape(np.tile(good_pixels, data_ped_charges.shape[0]),
data_ped_charges.shape)
# histogram of pedestal charges (biased extractor) from good and not noisy
# pixels:
qbins = 100
qrange = (-10, 15)
dataq = np.histogram(data_ped_charges[all_good].flatten(),
bins=qbins,
range=qrange,
density=True)
# Find the peak of the pedestal biased charge distribution of real data.
# Use an interpolated version of the histogram, for robustness:
func = interp1d(0.5 * (dataq[1][1:] + dataq[1][:-1]),
dataq[0],
kind='quadratic',
fill_value='extrapolate')
xx = np.linspace(qrange[0], qrange[1], 100 * qbins)
mode_data = xx[np.argmax(func(xx))]
# Event reader for simtel file:
mc_reader = EventSource(input_url=simtel_filename, config=Config(config))
# Obtain the configuration with which the pedestal calculations were
# performed:
ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator']
tel_id = ped_config['tel_id']
# Obtain the (unbiased) extractor used for pedestal calculations:
pedestal_extractor_type = ped_config['charge_product']
pedestal_calibrator = CameraCalibrator(
image_extractor_type=pedestal_extractor_type,
config=Config(ped_config),
subarray=mc_reader.subarray)
# Obtain the (usually biased) extractor used for shower images:
shower_extractor_type = config['image_extractor']
shower_calibrator = CameraCalibrator(
image_extractor_type=shower_extractor_type,
config=Config(config),
subarray=mc_reader.subarray)
# Since these extractors are now for use on MC, we have to apply the pulse
# integration correction (in data that is currently, as of
# lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the
# adc to pe conversion factors )
pedestal_calibrator.image_extractors[
ped_config['charge_product']].apply_integration_correction = True
shower_calibrator.image_extractors[
shower_extractor_type].apply_integration_correction = True
# Pulse integration window width of the (biased) extractor for showers:
shower_extractor_window_width = config[
config['image_extractor']]['window_width']
# Pulse integration window width for the pedestal estimation:
pedestal_extractor_config = ped_config[pedestal_extractor_type]
pedestal_extractor_window_width = pedestal_extractor_config['window_width']
# MC pedestals integrated with the unbiased pedestal extractor
mc_ped_charges = []
# MC pedestals integrated with the biased shower extractor
mc_ped_charges_biased = []
for event in mc_reader:
if tel_id not in event.trigger.tels_with_trigger:
continue
# Extract the signals as we do for pedestals (unbiased fixed window
# extractor):
pedestal_calibrator(event)
charges = event.dl1.tel[tel_id].image
# True number of pe's from Cherenkov photons (to identify noise-only pixels)
true_image = event.simulation.tel[tel_id].true_image
mc_ped_charges.append(charges[true_image == 0])
# Now extract the signal as we would do for shower events (usually
# with a biased extractor, e.g. LocalPeakWindowSum):
shower_calibrator(event)
charges_biased = event.dl1.tel[tel_id].image
mc_ped_charges_biased.append(charges_biased[true_image == 0])
# All pixels behave (for now) in the same way in MC, just put them together
mc_ped_charges = np.concatenate(mc_ped_charges)
mc_ped_charges_biased = np.concatenate(mc_ped_charges_biased)
mcq = np.histogram(mc_ped_charges_biased,
bins=qbins,
range=qrange,
density=True)
# Find the peak of the pedestal biased charge distribution of MC. Use
# an interpolated version of the histogram, for robustness:
func = interp1d(0.5 * (mcq[1][1:] + mcq[1][:-1]),
mcq[0],
kind='quadratic',
fill_value='extrapolate')
xx = np.linspace(qrange[0], qrange[1], 100 * qbins)
mode_mc = xx[np.argmax(func(xx))]
mc_unbiased_std_ped_pe = np.std(mc_ped_charges)
# Find the additional noise (in data w.r.t. MC) for the unbiased extractor,
# and scale it to the width of the window for integration of shower images.
# The idea is that when a strong signal is present, the biased extractor
# will integrate around it, and the additional noise is unbiased because
# it won't modify the integration range.
extra_noise_in_bright_pixels = \
((data_median_std_ped_pe**2 - mc_unbiased_std_ped_pe**2) *
shower_extractor_window_width / pedestal_extractor_window_width)
# Just in case, makes sure we just add noise if the MC noise is smaller
# than the real data's:
extra_noise_in_bright_pixels = max(0., extra_noise_in_bright_pixels)
bias = mode_data - mode_mc
extra_bias_in_dim_pixels = max(bias, 0)
# differences of values to peak charge:
dq = data_ped_charges[all_good].flatten() - mode_data
dqmc = mc_ped_charges_biased - mode_mc
# maximum distance (in pe) from peak, to avoid strong impact of outliers:
maxq = 10
# calculate widening of the noise bump:
added_noise = (np.sum(dq[dq < maxq]**2) / len(dq[dq < maxq]) -
np.sum(dqmc[dqmc < maxq]**2) / len(dqmc[dqmc < maxq]))
added_noise = (max(0, added_noise))**0.5
extra_noise_in_dim_pixels = added_noise
return extra_noise_in_dim_pixels, extra_bias_in_dim_pixels, \
extra_noise_in_bright_pixels
def prepare(self, particles, hillas=True, subarray=np.arange(600)):
''' Performs the processing of the images:
calibration perfoming conversion from r0 to dl1
image cleaning using tailcut_cleaning
image parametrization with Hillas parameters
Fills the dicts for clean_images and hillas moments
'''
# Input
#######
# particles - array of konsta_cta.readdata.FileReader
#
self.particles = particles
self.geoms, self.geoms_unique = self.get_camera_geoms()
# loop through all particle types
for particle in self.particles:
dtype = particle.datatype
# count number of images and number after cleaning
self.sum_images[dtype] = 0
self.sum_clean_images[dtype] = 0
# dict to store the hisograms
self.histograms = {}
# loop through all events
for event in particle.source:
event_id = event.dl0.event_id
n_images = 0
n_clean_images = 0
# count for each camera individually
images_cams = {}
clean_images_cams = {}
self.mc_energy[dtype, event_id] = event.mc.energy
# loop through all telescopes with data
for tel_id in event.r0.tels_with_data:
if type(subarray[0]) != str:
subarray = [str(x) for x in subarray]
if str(tel_id) in subarray:
pass
else:
continue
n_images += 1
# get camera information
camera = event.inst.subarray.tel[tel_id].camera
try:
# check if key already exists. if not initialize it with value 0
_im = images_cams[camera.cam_id]
_im = clean_images_cams[camera.cam_id]
except KeyError:
images_cams[camera.cam_id] = 0
clean_images_cams[camera.cam_id] = 0
# count for each camera individually
images_cams[camera.cam_id] += 1
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = self.integrator
cfg["ChargeExtractorFactory"][
"window_width"] = self.trace_width[camera.cam_id][0]
cfg["ChargeExtractorFactory"][
"window_shift"] = self.trace_width[camera.cam_id][1]
cfg['WaveformCleanerFactory']['product'] = self.cleaner
# usually all cameras are calibrated at once with the same camera.
# In order to force a racalculation to use the differen width and
# shift of the camera of this telescope, dl1 container will be reset
# to default at each iteration.
#
#Probably it is not even needed to reset this, as the container
#is refilled anyway.
event.dl1.tel[tel_id].reset()
# danger: The resulting calibrated event doesn't contain the right
# extracted charges for all cameras in the end as it seems like
# the dl1 images are overwirtten each time so that, the charges,
# extracted at the last telescope iteration will be contained in the
# dl1 container.
# As I'm wirting out all the required information for the telescope
# within this loop, this should not be much of a problem for now, but
# in the future a appropriate way to work arround this is required.
# set up calibrator.
calibrator = CameraCalibrator(r1_product=self.r1,
config=cfg)
# calibrate event
calibrator.calibrate(event)
######################################
# create output for comparison plots #
######################################
cam_id = camera.cam_id
number_gains = event.dl1.tel[tel_id].image.shape[0]
# create dict for the camera on first appearance
try:
_hist = self.histograms[cam_id]
except KeyError:
self.histograms[cam_id] = {}
# fill histograms and merge for all events dependent on gain
for gain, label in zip(range(number_gains),
["gain1", "gain2"]):
image = np.array(event.dl1.tel[tel_id].image[gain])
# only positive charges
hist_lower = len(image[image > np.power(10., -1.)])
hist_higher = len(image[image > np.power(10., 4.)])
image = image[image > 0]
logimage = np.log10(image)
hist = np.histogram(logimage, range=[-1, 4], bins=100)
# store the values outside of range for sanity check
_hist = np.append(hist_lower, hist[0])
_hist = np.append(_hist, hist_higher)
_bins = np.append(-1000, hist[1])
_bins = np.append(_bins, 1000)
hist = (_hist, _bins)
try:
self.histograms[cam_id][label] = (
self.histograms[cam_id][label][0] + hist[0],
self.histograms[cam_id][label][1])
except KeyError:
self.histograms[cam_id][label] = hist
#####################################
# Tino's solution for gain selection
if (camera.cam_id == np.array(list(
self.pe_thresh.keys()))).any():
image = event.dl1.tel[tel_id].image
image = self.pick_gain_channel(image, camera.cam_id)
else:
image = event.dl1.tel[tel_id].image
image = np.reshape(image[0], np.shape(image)[1])
######################
# image cleaning
# Threshold values adapted from Tino's repository
mask = tailcuts_clean(
self.geoms[tel_id],
image,
picture_thresh=self.tail_thresholds[camera.cam_id][1],
boundary_thresh=self.tail_thresholds[camera.cam_id][0],
min_number_picture_neighbors=0)
try:
temp_list
except NameError:
temp_list = []
if not (camera.cam_id in temp_list):
temp_list.append(camera.cam_id)
print("Threshold {camera}: {threshold}".format(
camera=camera.cam_id,
threshold=self.tail_thresholds[camera.cam_id]))
number_pixels = np.count_nonzero(mask)
# drop images that didn't survive image cleaning
if any(mask == True):
n_clean_images += 1
self.clean_images[dtype, event_id,
tel_id] = np.copy(image)
# set rejected pixels to zero
self.clean_images[dtype, event_id, tel_id][~mask] = 0
# count for each camera individually
try:
clean_images_cams[camera.cam_id] += 1
except KeyError:
clean_images_cams[camera.cam_id] = 1
### hillas parametrization
if hillas:
hillas_moments = hillas_parameters(
self.geoms[tel_id],
self.clean_images[dtype, event_id,
tel_id], True)
self.number_pixels.append(number_pixels)
self.energy.append(event.mc.energy.base)
self.size.append(hillas_moments.intensity)
self.length.append(hillas_moments.length)
self.width.append(hillas_moments.width)
self.skewness.append(hillas_moments.skewness)
self.camera.append(camera.cam_id)
self.core_x.append(event.mc.core_x.base)
self.core_y.append(event.mc.core_y.base)
else:
pass
# count number of images at trigge level and after cleaning
# summary:
self.sum_images[dtype] += n_images
self.sum_clean_images[dtype] += n_clean_images
# per event:
self.core_x2.append(event.mc.core_x.base)
self.core_y2.append(event.mc.core_y.base)
self.energy2.append(event.mc.energy.base)
self.n_images.append(float(n_images))
self.n_clean_images.append(float(n_clean_images))
for cam in images_cams.keys():
try:
self.images_cams[cam].append(float(images_cams[cam]))
self.clean_images_cams[cam].append(
float(clean_images_cams[cam]))
except:
self.images_cams[cam] = [float(images_cams[cam])]
self.clean_images_cams[cam] = [
float(clean_images_cams[cam])
]
print("Processed {} images for datatype {}. Images "
"that didn't survive cleaning: {}".format(
self.sum_images[dtype], dtype,
self.sum_images[dtype] - self.sum_clean_images[dtype]))
self.get_keys()
def r0_to_dl1(
input_filename=None,
output_filename=None,
custom_config={},
):
"""
Chain r0 to dl1
Save the extracted dl1 parameters in output_filename
Parameters
----------
input_filename: str
path to input file, default: `gamma_test_large.simtel.gz`
output_filename: str or None
path to output file, defaults to writing dl1 into the current directory
custom_config: path to a configuration file
Returns
-------
"""
# using None as default and using `get_dataset_path` only inside the function
# prevents downloading at import time.
if input_filename is None:
get_dataset_path('gamma_test_large.simtel.gz')
if output_filename is None:
try:
run = parse_r0_filename(input_filename)
output_filename = run_to_dl1_filename(run.tel_id, run.run,
run.subrun)
except ValueError:
output_filename = r0_to_dl1_filename(Path(input_filename).name)
if os.path.exists(output_filename):
raise IOError(str(output_filename) + ' exists, exiting.')
config = replace_config(standard_config, custom_config)
source = EventSource(input_url=input_filename,
config=Config(config["source_config"]))
subarray = source.subarray
is_simu = source.is_simulation
metadata = global_metadata()
write_metadata(metadata, output_filename)
cal_mc = load_calibrator_from_config(config, subarray)
# minimum number of pe in a pixel to include it
# in calculation of muon ring time (peak sample):
min_pe_for_muon_t_calc = 10.
# Dictionary to store muon ring parameters
muon_parameters = create_muon_table()
# all this will be cleaned up in a next PR related to the configuration files
r1_dl1_calibrator = CameraCalibrator(
image_extractor_type=config['image_extractor'],
config=Config(config),
subarray=subarray)
if not is_simu:
# Pulse extractor for muon ring analysis. Same parameters (window_width and _shift) as the one for showers, but
# using GlobalPeakWindowSum, since the signal for the rings is expected to be very isochronous
r1_dl1_calibrator_for_muon_rings = CameraCalibrator(
image_extractor_type=config['image_extractor_for_muons'],
config=Config(config),
subarray=subarray)
# Component to process interleaved pedestal and flat-fields
calib_config = Config({
config['calibration_product']:
config[config['calibration_product']]
})
calibration_calculator = CalibrationCalculator.from_name(
config['calibration_product'],
config=calib_config,
subarray=source.subarray)
calibration_index = DL1MonitoringEventIndexContainer()
dl1_container = DL1ParametersContainer()
extra_im = ExtraImageInfo()
extra_im.prefix = '' # get rid of the prefix
# Write extra information to the DL1 file
subarray.to_hdf(output_filename)
if is_simu:
write_mcheader(
source.simulation_config,
output_filename,
obs_id=source.obs_ids[0],
filters=HDF5_ZSTD_FILTERS,
metadata=metadata,
)
nsb_tuning = False
if 'waveform_nsb_tuning' in config.keys():
nsb_tuning = config['waveform_nsb_tuning']['nsb_tuning']
if nsb_tuning:
if is_simu:
nsb_original = extract_simulation_nsb(input_filename)
pulse_template = NormalizedPulseTemplate.load_from_eventsource(
subarray.tel[1].camera.readout)
if 'nsb_tuning_ratio' in config['waveform_nsb_tuning'].keys():
# get value from config to possibly extract it beforehand on multiple files for averaging purposes
# or gain time
nsb_tuning_ratio = config['waveform_nsb_tuning'][
'nsb_tuning_ratio']
else:
# extract the pedestal variance difference between the current MC file and the target data
# FIXME? fails for multiple telescopes
nsb_tuning_ratio = calculate_required_additional_nsb(
input_filename,
config['waveform_nsb_tuning']['target_data'],
config=config)[0]
spe = np.loadtxt(
config['waveform_nsb_tuning']['spe_location']).T
spe_integral = np.cumsum(spe[1])
charge_spe_cumulative_pdf = interp1d(spe_integral,
spe[0],
kind='cubic',
bounds_error=False,
fill_value=0.,
assume_sorted=True)
allowed_tel = np.zeros(len(nsb_original), dtype=bool)
allowed_tel[np.array(config['source_config']['LSTEventSource']
['allowed_tels'])] = True
logger.info('Tuning NSB on MC waveform from ' +
str(np.asarray(nsb_original)[allowed_tel]) +
'GHz to {0:d}%'.format(
int(nsb_tuning_ratio * 100 + 100.5)) +
' for telescopes ids ' +
str(config['source_config']['LSTEventSource']
['allowed_tels']))
else:
logger.warning(
'NSB tuning on waveform active in config but file is real data, option will be ignored'
)
nsb_tuning = False
with HDF5TableWriter(
filename=output_filename,
group_name='dl1/event',
mode='a',
filters=HDF5_ZSTD_FILTERS,
add_prefix=True,
# overwrite=True,
) as writer:
setup_writer(writer, source.subarray, is_simulation=is_simu)
event = None
for i, event in enumerate(source):
if i % 100 == 0:
logger.info(i)
event.dl0.prefix = ''
event.trigger.prefix = ''
if event.simulation is not None:
event.simulation.prefix = 'mc'
dl1_container.reset()
# write sub tables
if is_simu:
write_subarray_tables(writer, event, metadata)
cal_mc(event)
if config['mc_image_scaling_factor'] != 1:
rescale_dl1_charge(event,
config['mc_image_scaling_factor'])
else:
if i == 0:
# initialize the telescope
# FIXME? LST calibrator is only for one telescope
# it should be inside the telescope loop (?)
tel_id = calibration_calculator.tel_id
#initialize the event monitoring data
event.mon = deepcopy(source.r0_r1_calibrator.mon_data)
for container in [
event.mon.tel[tel_id].pedestal,
event.mon.tel[tel_id].flatfield,
event.mon.tel[tel_id].calibration
]:
add_global_metadata(container, metadata)
add_config_metadata(container, config)
# write the first calibration event (initialized from calibration h5 file)
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=True,
new_ff=True)
# flat-field or pedestal:
if (event.trigger.event_type == EventType.FLATFIELD
or event.trigger.event_type == EventType.SKY_PEDESTAL):
# process interleaved events (pedestals, ff, calibration)
new_ped_event, new_ff_event = calibration_calculator.process_interleaved(
event)
# write monitoring containers if updated
if new_ped_event or new_ff_event:
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped_event,
new_ff=new_ff_event)
# calibrate and gain select the event by hand for DL1
source.r0_r1_calibrator.calibrate(event)
# Option to add nsb in waveforms
if nsb_tuning:
# FIXME? assumes same correction ratio for all telescopes
for tel_id in config['source_config']['LSTEventSource'][
'allowed_tels']:
waveform = event.r1.tel[tel_id].waveform
readout = subarray.tel[tel_id].camera.readout
sampling_rate = readout.sampling_rate.to_value(u.GHz)
dt = (1.0 / sampling_rate)
selected_gains = event.r1.tel[tel_id].selected_gain_channel
mask_high = (selected_gains == 0)
tune_nsb_on_waveform(waveform, nsb_tuning_ratio,
nsb_original[tel_id] * u.GHz,
dt * u.ns, pulse_template, mask_high,
charge_spe_cumulative_pdf)
# create image for all events
r1_dl1_calibrator(event)
# Temporal volume reducer for lstchain - dl1 level must be filled and dl0 will be overwritten.
# When the last version of the method is implemented, vol. reduction will be done at dl0
apply_volume_reduction(event, subarray, config)
# FIXME? This should be eventually done after we evaluate whether the image is
# a candidate muon ring. In that case the full image could be kept, or reduced
# only after the ring analysis is complete.
for telescope_id, dl1_tel in event.dl1.tel.items():
dl1_tel.prefix = '' # don't really need one
tel_name = str(subarray.tel[telescope_id])[4:]
# extra info for the image table
extra_im.tel_id = telescope_id
extra_im.selected_gain_channel = event.r1.tel[
telescope_id].selected_gain_channel
add_global_metadata(extra_im, metadata)
add_config_metadata(extra_im, config)
focal_length = subarray.tel[
telescope_id].optics.equivalent_focal_length
mirror_area = subarray.tel[telescope_id].optics.mirror_area
dl1_container.reset()
# update the calibration index in the dl1 event container
dl1_container.calibration_id = calibration_index.calibration_id
dl1_container.fill_event_info(event)
# Will determine whether this event has to be written to the
# DL1 output or not.
if is_simu:
dl1_container.fill_mc(event,
subarray.positions[telescope_id])
assert event.dl1.tel[telescope_id].image is not None
try:
get_dl1(
event,
subarray,
telescope_id,
dl1_container=dl1_container,
custom_config=config,
)
except HillasParameterizationError:
logging.exception(
'HillasParameterizationError in get_dl1()')
if not is_simu:
dl1_container.ucts_time = 0
# convert Time to unix timestamp in (UTC) to keep compatibility
# with older lstchain
# FIXME: just keep it as time, table writer and reader handle it
dl1_container.dragon_time = event.trigger.time.unix
dl1_container.tib_time = 0
if 'ucts_jump' in vars(
event.lst.tel[telescope_id].evt.__class__):
dl1_container.ucts_jump = event.lst.tel[
telescope_id].evt.ucts_jump
dl1_container.ucts_trigger_type = event.lst.tel[
telescope_id].evt.ucts_trigger_type
dl1_container.trigger_type = event.lst.tel[
telescope_id].evt.tib_masked_trigger
else:
dl1_container.trigger_type = event.trigger.event_type
dl1_container.az_tel = event.pointing.tel[telescope_id].azimuth
dl1_container.alt_tel = event.pointing.tel[
telescope_id].altitude
dl1_container.trigger_time = event.trigger.time.unix
dl1_container.event_type = event.trigger.event_type
dl1_container.prefix = dl1_tel.prefix
for container in [extra_im, dl1_container, event.r0, dl1_tel]:
add_global_metadata(container, metadata)
add_config_metadata(container, config)
writer.write(table_name=f'telescope/parameters/{tel_name}',
containers=[event.index, dl1_container])
writer.write(table_name=f'telescope/image/{tel_name}',
containers=[event.index, dl1_tel, extra_im])
# Muon ring analysis, for real data only (MC is done starting from DL1 files)
if not is_simu:
bad_pixels = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
# Set to 0 unreliable pixels:
image = dl1_tel.image * (~bad_pixels)
# process only promising events, in terms of # of pixels with large signals:
if tag_pix_thr(image):
# re-calibrate r1 to obtain new dl1, using a more adequate pulse integrator for muon rings
numsamples = event.r1.tel[telescope_id].waveform.shape[
1] # not necessarily the same as in r0!
bad_pixels_hg = event.mon.tel[
telescope_id].calibration.unusable_pixels[0]
bad_pixels_lg = event.mon.tel[
telescope_id].calibration.unusable_pixels[1]
# Now set to 0 all samples in unreliable pixels. Important for global peak
# integrator in case of crazy pixels! TBD: can this be done in a simpler
# way?
bad_pixels = bad_pixels_hg | bad_pixels_lg
bad_waveform = np.transpose(
np.array(numsamples * [bad_pixels]))
# print('hg bad pixels:',np.where(bad_pixels_hg))
# print('lg bad pixels:',np.where(bad_pixels_lg))
event.r1.tel[telescope_id].waveform *= ~bad_waveform
r1_dl1_calibrator_for_muon_rings(event)
image = dl1_tel.image * (~bad_pixels)
# Check again: with the extractor for muon rings (most likely GlobalPeakWindowSum)
# perhaps the event is no longer promising (e.g. if it has a large time evolution)
if not tag_pix_thr(image):
good_ring = False
else:
# read geometry from event.inst. But not needed for every event. FIXME?
geom = subarray.tel[telescope_id].\
camera.geometry
muonintensityparam, dist_mask, \
ring_size, size_outside_ring, muonringparam, \
good_ring, radial_distribution, \
mean_pixel_charge_around_ring,\
muonpars = \
analyze_muon_event(subarray,
event.index.event_id,
image, geom, focal_length,
mirror_area, False, '')
# mirror_area, True, './')
# (test) plot muon rings as png files
# Now we want to obtain the waveform sample (in HG & LG) at which the ring light peaks:
bright_pixels = image > min_pe_for_muon_t_calc
selected_gain = event.r1.tel[
telescope_id].selected_gain_channel
mask_hg = bright_pixels & (selected_gain == 0)
mask_lg = bright_pixels & (selected_gain == 1)
bright_pixels_waveforms_hg = event.r1.tel[
telescope_id].waveform[mask_hg, :]
bright_pixels_waveforms_lg = event.r1.tel[
telescope_id].waveform[mask_lg, :]
stacked_waveforms_hg = np.sum(
bright_pixels_waveforms_hg, axis=0)
stacked_waveforms_lg = np.sum(
bright_pixels_waveforms_lg, axis=0)
# stacked waveforms from all bright pixels; shape (ngains, nsamples)
hg_peak_sample = np.argmax(stacked_waveforms_hg,
axis=-1)
lg_peak_sample = np.argmax(stacked_waveforms_lg,
axis=-1)
if good_ring:
fill_muon_event(-1, muon_parameters, good_ring,
event.index.event_id,
dl1_container.dragon_time,
muonintensityparam, dist_mask,
muonringparam, radial_distribution,
ring_size, size_outside_ring,
mean_pixel_charge_around_ring,
muonpars, hg_peak_sample,
lg_peak_sample)
# writes mc information per telescope, including photo electron image
if (is_simu and config['write_pe_image']
and event.simulation.tel[telescope_id].true_image
is not None and
event.simulation.tel[telescope_id].true_image.any()):
event.simulation.tel[telescope_id].prefix = ''
writer.write(table_name=f'simulation/{tel_name}',
containers=[
event.simulation.tel[telescope_id],
extra_im
])
if event is None:
logger.warning('No events in file')
if not is_simu and event is not None:
# at the end of event loop ask calculation of remaining interleaved statistics
new_ped, new_ff = calibration_calculator.output_interleaved_results(
event)
# write monitoring events
write_calibration_data(writer,
calibration_index,
event.mon.tel[tel_id],
new_ped=new_ped,
new_ff=new_ff)
if is_simu and event is not None:
# Reconstruct source position from disp for all events and write the result in the output file
add_disp_to_parameters_table(output_filename, dl1_params_lstcam_key,
focal_length)
# Write energy histogram from simtel file and extra metadata
# ONLY of the simtel file has been read until the end, otherwise it seems to hang here forever
if source.max_events is None:
write_simtel_energy_histogram(source,
output_filename,
obs_id=event.index.obs_id,
metadata=metadata)
if not is_simu:
dir, name = os.path.split(output_filename)
name = name.replace('dl1', 'muons').replace('LST-1.1', 'LST-1')
# Consider the possibilities of DL1 files with .fits.h5 & .h5 ending:
name = name.replace('.fits.h5', '.fits').replace('.h5', '.fits')
muon_output_filename = Path(dir, name)
table = Table(muon_parameters)
table.write(muon_output_filename, format='fits', overwrite=True)
def analyze_ctapipe():
print("starting ctapipe.")
gamma = FileReader(
"/lustre/fs21/group/cta/prod3b/prod3b-paranal20deg/gamma_onSource/gamma_20deg_0deg_run624___cta-prod3_desert-2150m-Paranal-merged.simtel.gz",
"gamma")
gamma.read_files()
trace_width = {
"ASTRICam": (10, 5),
"FlashCam": (4, 2),
"LSTCam": (4, 2),
"NectarCam": (6, 3),
"DigiCam": (4, 2),
"CHEC": (8, 4),
"SCTCam": (6, 3)
}
# from Tino
pe_thresh = {"ASTRICam": 14, "LSTCam": 100, "NectarCam": 190}
#integrators = ['GlobalPeakIntegrator', 'LocalPeakIntegrator', 'NeighbourPeakIntegrator', 'AverageWfPeakIntegrator']
#cleaners = ['NullWaveformCleaner', 'CHECMWaveformCleanerAverage', 'CHECMWaveformCleanerLocal']
for integrator in ['NeighbourPeakIntegrator']:
i = 0
for event in gamma.source:
i += 1
if not event.r0.event_id in event_IDs:
# continue with next event
continue
else:
for tel in event.r0.tels_with_data:
camera = event.inst.subarray.tel[tel].camera
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = integrator
cfg["ChargeExtractorFactory"][
"window_width"] = trace_width[camera.cam_id][0]
cfg["ChargeExtractorFactory"][
"window_shift"] = trace_width[camera.cam_id][1]
event.dl1.tel[tel].reset()
# set up calibrator.
calibrator = CameraCalibrator(
r1_product="HESSIOR1Calibrator", config=cfg)
calibrator.calibrate(event)
#####################################
# Tino's solution for gain selection
if (camera.cam_id == np.array(list(
pe_thresh.keys()))).any():
image = event.dl1.tel[tel].image
image = pick_gain_channel(image, camera.cam_id,
pe_thresh)
else:
image = event.dl1.tel[tel].image
image = np.reshape(image[0], np.shape(image)[1])
# image = np.reshape(image[0], np.shape(image)[1])
EventID = [int(event.r0.event_id)] * len(image)
TelescopeID = [int(tel)] * len(image)
cameras = [event.inst.subarray.tel[tel].camera.cam_id
] * len(image)
PixelID = np.arange(len(image))
charge = image
new_df = pd.DataFrame(np.transpose(
[EventID, TelescopeID, PixelID, charge, cameras]),
columns=[
"EventID", "TelescopeID",
"PixelID", "charge", "cameras"
])
try:
ctapipe_charge = ctapipe_charge.append(
new_df, ignore_index=True)
except (TypeError, NameError):
ctapipe_charge = new_df
print("{:.2f}% finished".format((i / len(event_IDs)) * 100))
ctapipe_charge.to_csv('ctapipe_charge.csv', sep=",")
return ctapipe_charge
def setup(self):
self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
data_config = self.config.copy()
data_config['WaveformCleanerFactory'] = Config(cleaner='CHECMWaveformCleanerLocal')
mc_config = self.config.copy()
data_kwargs = dict(config=data_config, tool=self)
mc_kwargs = dict(config=mc_config, tool=self)
filepath = '/Volumes/gct-jason/data/170330/onsky-mrk501/Run05477_r1.tio'
reader = TargetioFileReader(input_path=filepath, **data_kwargs)
filepath = '/Users/Jason/Software/outputs/sim_telarray/meudon_cr/simtel_proton_nsb50_thrs30_1petal_rndm015_heide.gz'
# filepath = '/Users/Jason/Software/outputs/sim_telarray/meudon_cr/simtel_proton_nsb50_thrs30.gz'
reader_mc = HessioFileReader(input_path=filepath, **mc_kwargs)
calibrator = CameraCalibrator(origin=reader.origin,
**data_kwargs)
calibrator_mc = CameraCalibrator(origin=reader_mc.origin,
**mc_kwargs)
first_event = reader.get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
pos = first_event.inst.pixel_pos[telid]
foclen = first_event.inst.optical_foclen[telid]
geom = CameraGeometry.guess(*pos, foclen)
first_event = reader_mc.get_event(0)
telid = list(first_event.r0.tels_with_data)[0]
pos_mc = first_event.inst.pixel_pos[telid]
foclen = first_event.inst.optical_foclen[telid]
geom_mc = CameraGeometry.guess(*pos_mc, foclen)
d1 = dict(type='Data', reader=reader, calibrator=calibrator,
pos=pos, geom=geom, t1=20, t2=10)
d2 = dict(type='MC', reader=reader_mc, calibrator=calibrator_mc,
pos=pos_mc, geom=geom_mc, t1=20, t2=10)
self.reader_df = pd.DataFrame([d1, d2])
p_kwargs = data_kwargs
p_kwargs['script'] = "checm_paper_hillas"
p_kwargs['figure_name'] = "all_images"
self.p_allimage = AllImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "all_peak_time_images"
self.p_alltimeimage = PeakTimePlotter(**p_kwargs)
p_kwargs['figure_name'] = "all_mc_images"
self.p_allmcimage = AllImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "zero_width_images"
self.p_zwimage = ZeroWidthImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "zero_width_mc_images"
self.p_zwmcimage = ZeroWidthImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "muon_images"
self.p_muonimage = MuonImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "bright_images"
self.p_brightimage = BrightImagePlotter(**p_kwargs)
p_kwargs['figure_name'] = "count_image"
self.p_countimage = CountPlotter(**p_kwargs)
p_kwargs['figure_name'] = "whole_distribution"
self.p_whole_dist = WholeDist(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "width_vs_length"
self.p_widthvslength = WidthVsLength(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "size_vs_length"
self.p_sizevslength = SizeVsLength(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "width_div_length"
self.p_widthdivlength = WidthDivLength(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "length_div_size"
self.p_lengthdivsize = LengthDivSize(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "pair_plot"
self.p_pair = PairPlotter(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "pair_mc_plot"
self.p_mc_pair = PairPlotter(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "length"
self.p_length = LengthPlotter(**p_kwargs, shape='wide')
p_kwargs['figure_name'] = "width"
self.p_width = WidthPlotter(**p_kwargs, shape='wide')
def test_manual_extractor():
calibrator = CameraCalibrator(extractor_name="LocalPeakWindowSum")
assert isinstance(calibrator.dl1.extractor, LocalPeakWindowSum)