def test_stage1():
"""Test the ctapipe stage1 tool can read in LST real data using the event source"""
from ctapipe.tools.process import ProcessorTool
from ctapipe.core import run_tool
tool = ProcessorTool()
output = str(test_cal_path).replace(".root", ".h5")
ret = run_tool(tool, argv=[
f'--input={test_cal_path}',
f'--output={output}',
f'--config={str(config)}',
"--camera-frame",
])
assert ret == 0
parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001')
assert len(parameters) == 458
trigger = read_table(output, '/dl1/event/subarray/trigger')
event_type_counts = np.bincount(trigger['event_type'])
# no pedestals expected, should be only physics data
assert event_type_counts.sum() == 458
assert event_type_counts[EventType.FLATFIELD.value] == 0
assert event_type_counts[EventType.SKY_PEDESTAL.value] == 0
assert event_type_counts[EventType.SUBARRAY.value] == 458
def test_image_modifications(tmp_path, dl1_image_file):
"""
Test that running ctapipe-process with an ImageModifier set
produces a file with different images.
"""
unmodified_images = read_table(dl1_image_file,
"/dl1/event/telescope/images/tel_025")
noise_config = resource_file("image_modification_config.json")
dl1_modified = tmp_path / "dl1_modified.dl1.h5"
assert (run_tool(
ProcessorTool(),
argv=[
f"--config={noise_config}",
f"--input={dl1_image_file}",
f"--output={dl1_modified}",
"--write-parameters",
"--overwrite",
],
cwd=tmp_path,
) == 0)
modified_images = read_table(dl1_modified,
"/dl1/event/telescope/images/tel_025")
# Test that significantly more light is recorded (bias in dim pixels)
assert modified_images["image"].sum() / unmodified_images["image"].sum(
) > 1.5
def read_dl1(path, images=False, tel_id="LST_LSTCam", root="dl1"):
if isinstance(tel_id, int):
tel = f"tel_{tel_id:03d}"
else:
tel = tel_id
events = read_table(
path,
f"/dl1/event/telescope/parameters/{tel}") #, start=0, stop=1000000)
# lstchain has a different scheme, lets just not use these for now
# pointing = read_table(path, f"/dl1/monitoring/telescope/pointing/{tel}")
# trigger = read_table(path, "/dl1/event/telescope/trigger")
if images:
images = read_table(path, f"/dl1/event/telescope/images/{tel}")
assert len(events) == len(images)
events = join(events,
images,
keys=["obs_id", "event_id"],
join_type="left")
# there are no magic numbers here. move on
# events["tel_id"] = tel_id
# events = join(
# events, trigger, keys=["obs_id", "event_id", "tel_id"], join_type="left"
# )
# that changed at some point
if "time" in events.keys():
time_key = "time"
elif "dragon_time" in events.keys():
time_key = "dragon_time"
events["time"] = Time(events[time_key], format="unix")
else:
time_key = "trigger_time"
events["time"] = Time(events[time_key], format="unix")
time_key = "time"
return events
# events = join(events, pointing, keys=time_key, join_type="left")
# masked columns make everything harder
events = events.filled(np.nan)
alt_key = "altitude" if "altitude" in events.keys() else "alt_tel"
az_key = "azimuth" if "azimuth" in events.keys() else "az_tel"
events["azimuth"] = np.interp(events[time_key].mjd, pointing[time_key].mjd,
pointing["az_key"].quantity.to_value(
u.deg)) * u.deg
events["alt_key"] = np.interp(
events[time_key].mjd, pointing[time_key].mjd,
pointing["altitude"].quantity.to_value(u.deg)) * u.deg
#events["azimuth"] = ffill(events["azimuth"])
#events["altitude"] = ffill(events["altitude"])
# done in ctapipe?
#events = add_units(events)
events["time"] = Time(events[time_key], format="mjd", scale="tai")
# this is failing because of broken header info for some reason
subarray = SubarrayDescription.from_hdf(path)
events["focal_length"] = subarray.tels[1].optics.equivalent_focal_length
#events["focal_length"] = 28 * u.m
return events
def read_sim_info(path):
key = "/simulation/run_config"
log.info(f"Reading sim info for file {path} and key {key}")
run_info = read_table(path, key)
e_min = np.unique(run_info.columns["energy_range_min"])
assert len(e_min) == 1
e_max = np.unique(run_info.columns["energy_range_max"])
assert len(e_max) == 1
max_impact = np.unique(run_info.columns["max_scatter_range"])
assert len(max_impact) == 1
index = np.unique(run_info.columns["spectral_index"])
assert len(index) == 1
view_cone = np.unique(run_info.columns["max_viewcone_radius"])
assert len(view_cone) == 1
sim_info = SimulatedEventsInfo(
n_showers=(run_info.columns["shower_reuse"] *
run_info.columns["num_showers"]).sum(),
energy_min=u.Quantity(e_min[0], u.TeV),
energy_max=u.Quantity(e_max[0], u.TeV),
max_impact=u.Quantity(max_impact[0], u.m),
spectral_index=index[0],
viewcone=u.Quantity(view_cone[0], u.deg),
)
return sim_info
def test_strings(tmp_path):
"""Test we can write unicode strings"""
from ctapipe.core import Container
from ctapipe.io import read_table
# when not giving a max_len, should be taken from the first container
class Container1(Container):
container_prefix = ""
string = Field("", "test string")
path = tmp_path / "test.h5"
strings = ["Hello", "öäα"]
with HDF5TableWriter(path, mode="w") as writer:
for string in strings:
writer.write("strings", Container1(string=string))
table = read_table(path, "/strings")
# the α is above the max length estimated from the first element
assert table["string"].tolist() == ["Hello", "öä"]
class Container2(Container):
container_prefix = ""
string = Field("", "test string", max_length=10)
path = tmp_path / "test.h5"
strings = ["Hello", "öäα", "12345678910"]
expected = ["Hello", "öäα", "1234567891"]
with HDF5TableWriter(path, mode="w") as writer:
for string in strings:
writer.write("strings", Container2(string=string))
table = read_table(path, "/strings")
# the α is above the max length estimated from the first element
assert table["string"].tolist() == expected
# test this also works with table reader
with HDF5TableReader(path) as reader:
generator = reader.read("/strings", Container2)
for string in expected:
c = next(generator)
assert c.string == string
def test_lstchain_find_pedestals(temp_dir_observed_files, observed_dl1_files):
run_program(
"lstchain_find_pedestals",
"--input-dir",
temp_dir_observed_files,
temp_dir_observed_files,
)
for subrun, expected_length in zip((0, 100), (0, 1)):
path = temp_dir_observed_files / f"pedestal_ids_Run02008.{subrun:04d}.h5"
assert path.is_file()
t = read_table(path, "/interleaved_pedestal_ids")
assert len(t) == expected_length
def test_append_container(tmp_path):
path = tmp_path / "test_append.h5"
with HDF5TableWriter(path, mode="w") as writer:
for event_id in range(10):
hillas = HillasParametersContainer()
index = TelEventIndexContainer(obs_id=1, event_id=event_id, tel_id=1)
writer.write("data", [index, hillas])
with HDF5TableWriter(path, mode="a") as writer:
for event_id in range(10):
index = TelEventIndexContainer(obs_id=2, event_id=event_id, tel_id=1)
hillas = HillasParametersContainer()
writer.write("data", [index, hillas])
table = read_table(path, "/data")
assert len(table) == 20
assert np.all(table["obs_id"] == np.repeat([1, 2], 10))
assert np.all(table["event_id"] == np.tile(np.arange(10), 2))
def read_mc_dl1(path,
drop_nans=True,
rename=True,
images=False,
tel_id="LST_LSTCam"):
events = read_dl1(path, images=images, tel_id=tel_id, root="simulation")
log.info(events.keys())
if drop_nans:
events = remove_nans(events)
if not "mc_energy" in events.keys():
mc = read_table(path, "/simulation/event/subarray/shower")
mc = add_units(mc)
events = join(events,
mc,
join_type="left",
keys=["obs_id", "event_id"])
if rename:
return rename_columns(events)
return events
def test_create_drs4_pedestal_file(tmp_path):
'''Test the lstchain_drs4_pedestal_file tool'''
from lstchain.tools.lstchain_create_drs4_pedestal_file import DRS4PedestalAndSpikeHeight
input_path = test_data / "real/R0/20200218/LST-1.1.Run02005.0000_first50.fits.fz"
output_path = tmp_path / "drs4_pedestal_02005.h5"
ret = run_tool(
DRS4PedestalAndSpikeHeight(),
argv=[
f"--input={input_path}",
f"--output={output_path}",
"--overwrite",
],
cwd=tmp_path,
)
assert ret == 0, 'Running DRS4PedestalAndSpikeHeight tool failed'
assert output_path.is_file(), 'Output file not written'
drs4_data = read_table(output_path,
'/r1/monitoring/drs4_baseline/tel_001')[0]
baseline_mean = drs4_data['baseline_mean']
baseline_counts = drs4_data['baseline_std']
assert baseline_mean.dtype == np.uint16
assert baseline_mean.shape == (N_GAINS, N_PIXELS, N_CAPACITORS_PIXEL)
assert np.isclose(np.average(baseline_mean[baseline_counts > 0],
weights=baseline_counts[baseline_counts > 0]),
400,
rtol=0.05)
spike_height = drs4_data['spike_height']
assert spike_height.dtype == np.uint16
mean_spike_height = np.nanmean(spike_height, axis=(0, 1))
# these are the expected spike heights, but due to the low statistics,
# we need to use a rather large atol
assert np.allclose(mean_spike_height, [46, 53, 7], atol=2)
def main():
args = parser.parse_args()
files = sorted(args.srcdir.glob('dl1_LST-1.Run?????.????.h5'))
if not files:
raise IOError("No input dl1 files found")
output_dir = args.output_dir.absolute()
output_dir.mkdir(exist_ok=True, parents=True)
for file in files:
run_info = parse_dl1_filename(file)
run_number, subrun_index = run_info.run, run_info.subrun
# Approximate interleaved pedestal frequency in Hz. This changed once (so far) in the life of LST.
# We just use the run number to decide which frequency we should expect:
approximate_frequency = 50
if run_number > 2708:
approximate_frequency = 100
table = read_table(file, '/dl1/event/telescope/parameters/LST_LSTCam')
all_times = np.array(table['dragon_time'])
# probable interleaved flat-field events (just in case tag is
# faulty!):
ffmask = ((table['intensity'] > 3e4) &
(table['concentration_pixel'] < 0.005))
pedmask = np.zeros(len(table), dtype=bool)
pedmask[~ffmask] = find_pedestals(all_times[~ffmask],
approximate_frequency)
# Now remove the 10 brightest events (might be cosmics accidentally
# falling in the time windows determined by time_pedestals). The
# expected value of cosmics is smaller that, so we are probably
# removing some pedestals, but it does not harm.
intensity = np.array(table['intensity'])
# set nans to 0, to avoid problems with sorting:
intensity[np.isnan(intensity)] = 0
decreasing_intensity_ordered_indices = np.flip(np.argsort(intensity))
pedmask2 = pedmask[decreasing_intensity_ordered_indices]
# indices of likely pedestals, ordered from brightest to dimmest:
remove_indices = decreasing_intensity_ordered_indices[pedmask2]
pedmask[remove_indices[:10]] = False
n_pedestals = np.count_nonzero(pedmask)
pedestal_rate = n_pedestals / (all_times[-1] - all_times[0])
print(file)
print(f' Rate of identified pedestals: {pedestal_rate:.3f} Hz')
if n_pedestals > 0:
print(
f' Maximum intensity: {np.nanmax(intensity[pedmask]):.1f} pe')
else:
print(' Did not find any pedestal events')
output_file = Path(
output_dir,
f'pedestal_ids_Run{run_number:0>5}.{subrun_index:0>4}.h5')
data = table[pedmask][['obs_id', 'event_id']].as_array()
with tables.open_file(output_file, "w") as outfile:
outfile.create_table(
"/",
"interleaved_pedestal_ids",
obj=data,
)
def test_no_ff_tagging(tmpdir):
"""Test the ctapipe stage1 tool can read in LST real data using the event source"""
from ctapipe.tools.stage1 import Stage1Tool
from ctapipe.core.tool import run_tool
tmpdir = Path(tmpdir)
config_path = tmpdir / 'config.json'
config = {
'LSTEventSource': {
"use_flatfield_heuristic": False,
'LSTR0Corrections': {
'drs4_pedestal_path': str(test_drs4_pedestal_path),
'drs4_time_calibration_path': str(test_time_calib_path),
'calibration_path': str(test_calib_path),
},
'PointingSource': {
'drive_report_path': str(test_drive_report)
},
'EventTimeCalculator': {
'run_summary_path': str(test_run_summary),
},
},
"CameraCalibrator": {
"image_extractor_type": "LocalPeakWindowSum",
"LocalPeakWindowSum": {
"window_shift": 4,
"window_width": 8,
"apply_integration_correction": False,
}
},
"TailcutsImageCleaner": {
"picture_threshold_pe": 6,
"boundary_threshold_pe": 3,
"keep_isolated_pixels": False,
"min_picture_neighbors": 1,
}
}
with config_path.open('w') as f:
json.dump(config, f)
tool = Stage1Tool()
output = tmpdir / "test_dl1.h5"
ret = run_tool(tool,
argv=[
f'--input={test_r0_path}',
f'--output={output}',
f'--config={config_path}',
])
assert ret == 0
# test our custom default works
assert tool.event_source.r0_r1_calibrator.gain_selector.threshold == 3500
parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001')
assert len(parameters) == 200
trigger = read_table(output, '/dl1/event/subarray/trigger')
# test regression of event time calculation
first_event_time = Time(59101.95035244, format='mjd', scale='tai')
assert np.all((trigger['time'] - first_event_time).to_value(u.s) < 10)
event_type_counts = np.bincount(trigger['event_type'])
# one pedestal and flat field expected each, rest should be physics data
# without ff heuristic, the ff event will have type SUBARRAY
assert event_type_counts.sum() == 200
assert event_type_counts[EventType.FLATFIELD.value] == 0
assert event_type_counts[EventType.SKY_PEDESTAL.value] == 1
assert event_type_counts[EventType.SUBARRAY.value] == 199
def read_pedestal(path, tel_id):
'''Read the pedestal array from a h5 file'''
table = read_table(path, f"/r1/monitoring/drs4_baseline/tel_{tel_id:03d}")
return table[0]['baseline_mean']
def read_cta_dl1(path, aict_config, key=None, columns=None, first=None, last=None):
from ctapipe.io import read_table
from astropy.table import Table, join, vstack
import astropy.units as u
# choose the telescope tables to load
with tables.open_file(path) as file_table:
if key:
tels_to_load = (key,)
print(path, key)
else:
if aict_config.telescopes:
tels_to_load = [
f"/dl1/event/telescope/parameters/{tel.name}"
for tel in file_table.root.dl1.event.telescope.parameters
if tel.name in aict_config.telescopes
]
else:
tels_to_load = [
f"/dl1/event/telescope/parameters/{tel.name}"
for tel in file_table.root.dl1.event.telescope.parameters
]
if "equivalent_focal_length" in columns:
layout = Table.read(path, "/configuration/instrument/subarray/layout")
optics = Table.read(path, "/configuration/instrument/telescope/optics")
optics["tel_description"] = optics["description"]
optics.meta.clear()
layout = join(
layout,
optics[["tel_description", "equivalent_focal_length"]],
join_type="left",
keys="tel_description",
)
# load the telescope parameter table(s)
tel_tables = []
for tel in tels_to_load:
log.info(f'Loading data for telescope {tel}')
# as not all columns are located here, we cant just use
# columns=columns
tel_table = read_table(path, tel)[first:last]
# astropy table joins do not preserve input order
# but sort by the join keys, we add this simple index
# so we can restore the input order after joining
tel_table["__index"] = np.arange(len(tel_table))
# Pointing information has to be loaded from the monitoring tables and interpolated
# We also need the trigger tables as monitoring is based on time not events
if columns:
if "azimuth" in columns or "altitude" in columns:
tel_key = tel.split("/")[-1]
tel_triggers = read_table(path, "/dl1/event/telescope/trigger")
tel_table = join(
tel_table,
tel_triggers,
join_type="inner",
keys=["obs_id", "event_id", "tel_id"],
)
tel_pointings = read_table(
path, f"/dl1/monitoring/telescope/pointing/{tel_key}"
)
if aict_config.datamodel_version > "1.0.0":
time_key = "time"
else:
time_key = "telescopetrigger_time"
tel_table["azimuth"] = np.interp(
tel_table[time_key].mjd,
tel_pointings[time_key].mjd,
tel_pointings["azimuth"].quantity.to_value(u.deg),
) * u.deg
tel_table["altitude"] = np.interp(
tel_table[time_key].mjd,
tel_pointings[time_key].mjd,
tel_pointings["altitude"].quantity.to_value(u.deg),
) * u.deg
if "equivalent_focal_length" in columns:
tel_table = join(
tel_table,
layout[["tel_id", "equivalent_focal_length"]],
join_type="left",
keys="tel_id",
)
if columns:
# True / Simulation columns are still missing, so only use the columns already present
tel_table = tel_table[
list(set(columns).intersection(tel_table.columns)) + ['__index']
].copy()
# restore the input order that was changed by astropy.table.join
# and remove the additional column
tel_table.sort('__index')
tel_table.remove_column('__index')
tel_tables.append(tel_table)
# Monte carlo information is located in the simulation group
# and we are interested in the array wise true information only
event_table = vstack(tel_tables)
event_table = convert_units(event_table, aict_config)
df = pd.DataFrame(event_table.as_array()) # workaround for #11286 in astropy 4.2
if columns:
true_columns = [x for x in columns if x.startswith("true")]
if true_columns:
true_information = read_table(
path, "/simulation/event/subarray/shower"
)[true_columns + ["obs_id", "event_id"]]
true_information = true_information.to_pandas()
df = df.merge(true_information, on=["obs_id", "event_id"], how="left")
return df
def read_dl3(path):
events = read_table(path, 'EVENTS')
pointings = read_table(path, 'POINTING')
return join(events, pointings, keys='TIME')
def calculate_required_additional_nsb(simtel_filename,
data_dl1_filename,
config=None):
# TODO check if good estimation
# TODO reduce duplicated code with 'calculate_noise_parameters'
"""
Calculates the additional NSB needed in the MC waveforms
to match a real data DL1 file
Parameters
----------
simtel_filename: a simtel file containing showers, from the production
(same NSB and telescope settings) as the one on which the correction will
be applied. It must contain pixel-wise info on true number of p.e.'s from
C-photons (will be used to identify pixels which only contain noise).
data_dl1_filename: a real data DL1 file (processed with calibration
settings corresponding to those with which the MC is to be processed).
It must contain calibrated images, i.e. "DL1a" data. This file has the
"target" NSB which we want to have in the MC files, for better
agreement of data and simulations.
config: configuration containing the calibration
settings used for processing both the data and the MC files above
Returns
-------
extra_nsb: Fraction of the additional NSB in data compared to MC.
data_ped_variance: Pedestal variance from data
mc_ped_variance: Pedestal variance from MC
"""
log.setLevel(logging.INFO)
if config is None:
config = standard_config
# Real data DL1 tables:
data_dl1_calibration = read_table(
data_dl1_filename, '/dl1/event/telescope/monitoring/calibration')
data_dl1_pedestal = read_table(data_dl1_filename,
'/dl1/event/telescope/monitoring/pedestal')
unusable = data_dl1_calibration['unusable_pixels']
# Locate pixels with HG declared unusable either in original calibration or
# in interleaved events:
bad_pixels = unusable[0][0] # original calibration
for tf in unusable[1:][0]: # calibrations with interleaved
bad_pixels = np.logical_or(bad_pixels, tf)
good_pixels = ~bad_pixels
# First index: 1,2,... = values from interleaved (0 is for original
# calibration run)
# Second index: 0 = high gain
# Third index: pixels
# HG adc to pe conversion factors from interleaved calibrations:
data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :]
# Pixel-wise pedestal standard deviation (for an unbiased extractor),
# in adc counts:
data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :]
# indices which connect each pedestal calculation to a given calibration:
calibration_id = data_dl1_pedestal['calibration_id'][1:]
# convert pedestal st deviations to p.e.
dummy = []
for i, x in enumerate(data_HG_ped_std[:, ]):
dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ])
dummy = np.array(dummy)
# Average for all interleaved calibrations (in case there are more than one)
data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel
# Identify noisy pixels, likely containing stars - we want to adjust MC to
# the average diffuse NSB across the camera
data_median_std_ped_pe = np.median(data_HG_ped_std_pe)
data_std_std_ped_pe = np.std(data_HG_ped_std_pe)
log.info(f'Real data: median across camera of good pixels\' pedestal std '
f'{data_median_std_ped_pe:.3f} p.e.')
brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe
too_bright_pixels = (data_HG_ped_std_pe > brightness_limit)
log.info(f'Number of pixels beyond 3 std dev of median: '
f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)')
# Exclude too bright pixels, besides those with unusable calibration:
good_pixels &= ~too_bright_pixels
# recalculate the median of the pixels' std dev, with good_pixels:
data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels])
log.info(f'Good and not too bright pixels: {good_pixels.sum()}')
# Event reader for simtel file:
mc_reader = EventSource(input_url=simtel_filename, config=Config(config))
# Obtain the configuration with which the pedestal calculations were
# performed:
ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator']
tel_id = ped_config['tel_id']
# Obtain the (unbiased) extractor used for pedestal calculations:
pedestal_calibrator = CameraCalibrator(
image_extractor_type=ped_config['charge_product'],
config=Config(config['LSTCalibrationCalculator']),
subarray=mc_reader.subarray)
# Since these extractors are now for use on MC, we have to apply the pulse
# integration correction (in data that is currently, as of
# lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the
# adc to pe conversion factors )
pedestal_calibrator.image_extractors[
ped_config['charge_product']].apply_integration_correction = True
# MC pedestals integrated with the unbiased pedestal extractor
mc_ped_charges = []
for event in mc_reader:
if tel_id not in event.trigger.tels_with_trigger:
continue
# Extract the signals as we do for pedestals (unbiased fixed window
# extractor):
pedestal_calibrator(event)
charges = event.dl1.tel[tel_id].image
# True number of pe's from Cherenkov photons (to identify noise-only pixels)
true_image = event.simulation.tel[tel_id].true_image
mc_ped_charges.append(charges[true_image == 0])
# All pixels behave (for now) in the same way in MC, just put them together
mc_ped_charges = np.concatenate(mc_ped_charges)
mc_unbiased_std_ped_pe = np.std(mc_ped_charges)
# Find the additional noise (in data w.r.t. MC) for the unbiased extractor
# The idea is that pedestal variance scales with NSB
data_ped_variance = data_median_std_ped_pe**2
mc_ped_variance = mc_unbiased_std_ped_pe**2
extra_nsb = ((data_ped_variance - mc_ped_variance) / mc_ped_variance)
return extra_nsb, data_ped_variance, mc_ped_variance
def calculate_noise_parameters(simtel_filename,
data_dl1_filename,
config_filename=None):
"""
Calculates the parameters needed to increase the noise in an MC DL1 file
to match the noise in a real data DL1 file, using add_noise_in_pixels
The returned parameters are those needed by the function add_noise_in_pixels (see
description in its documentation above).
Parameters
----------
simtel_filename: `str`
a simtel file containing showers, from the same
production (same NSB and telescope settings) as the MC DL1 file below. It
must contain pixel-wise info on true number of p.e.'s from C-photons (
will be used to identify pixels which only contain noise).
data_dl1_filename: `str`
a real data DL1 file (processed with calibration
settings corresponding to those with which the MC is to be processed).
It must contain calibrated images, i.e. "DL1a" data. This file has the
"target" noise which we want to have in the MC files, for better
agreement of data and simulations.
config_filename: `str`
configuration file containing the calibration
settings used for processing both the data and the MC files above
Returns
-------
extra_noise_in_dim_pixels: `float`
Extra noise of dim pixels.
extra_bias_in_dim_pixels: `float`
Extra bias of dim pixels.
extra_noise_in_bright_pixels: `float`
Extra noise of bright pixels
"""
log.setLevel(logging.INFO)
if config_filename is None:
config = standard_config
else:
config = read_configuration_file(config_filename)
# Real data DL1 tables:
data_dl1_calibration = read_table(
data_dl1_filename, '/dl1/event/telescope/monitoring/calibration')
data_dl1_pedestal = read_table(data_dl1_filename,
'/dl1/event/telescope/monitoring/pedestal')
data_dl1_parameters = read_table(
data_dl1_filename, '/dl1/event/telescope/parameters/LST_LSTCam')
data_dl1_image = read_table(data_dl1_filename,
'/dl1/event/telescope/image/LST_LSTCam')
unusable = data_dl1_calibration['unusable_pixels']
# Locate pixels with HG declared unusable either in original calibration or
# in interleaved events:
bad_pixels = unusable[0][0] # original calibration
for tf in unusable[1:][0]: # calibrations with interleaveds
bad_pixels = np.logical_or(bad_pixels, tf)
good_pixels = ~bad_pixels
# First index: 1,2,... = values from interleaveds (0 is for original
# calibration run)
# Second index: 0 = high gain
# Third index: pixels
# HG adc to pe conversion factors from interleaved calibrations:
data_HG_dc_to_pe = data_dl1_calibration['dc_to_pe'][:, 0, :]
# Pixel-wise pedestal standard deviation (for an unbiased extractor),
# in adc counts:
data_HG_ped_std = data_dl1_pedestal['charge_std'][1:, 0, :]
# indices which connect each pedestal calculation to a given calibration:
calibration_id = data_dl1_pedestal['calibration_id'][1:]
# convert pedestal st deviations to p.e.
dummy = []
for i, x in enumerate(data_HG_ped_std[:, ]):
dummy.append(x * data_HG_dc_to_pe[calibration_id[i], ])
dummy = np.array(dummy)
# Average for all interleaved calibrations (in case there are more than one)
data_HG_ped_std_pe = np.mean(dummy, axis=0) # one value per pixel
# Identify noisy pixels, likely containing stars - we want to adjust MC to
# the average diffuse NSB across the camera
data_median_std_ped_pe = np.median(data_HG_ped_std_pe)
data_std_std_ped_pe = np.std(data_HG_ped_std_pe)
log.info(f'Real data: median across camera of good pixels\' pedestal std '
f'{data_median_std_ped_pe:.3f} p.e.')
brightness_limit = data_median_std_ped_pe + 3 * data_std_std_ped_pe
too_bright_pixels = (data_HG_ped_std_pe > brightness_limit)
log.info(f'Number of pixels beyond 3 std dev of median: '
f'{too_bright_pixels.sum()}, (above {brightness_limit:.2f} p.e.)')
ped_mask = data_dl1_parameters['event_type'] == 2
# The charges in the images below are obtained with the extractor for
# showers, usually a biased one, like e.g. LocalPeakWindowSum
data_ped_charges = data_dl1_image['image'][ped_mask]
# Exclude too bright pixels, besides those with unusable calibration:
good_pixels &= ~too_bright_pixels
# recalculate the median of the pixels' std dev, with good_pixels:
data_median_std_ped_pe = np.median(data_HG_ped_std_pe[good_pixels])
log.info(f'Good and not too bright pixels: {good_pixels.sum()}')
# all_good is an events*pixels boolean array of valid signals:
all_good = np.reshape(np.tile(good_pixels, data_ped_charges.shape[0]),
data_ped_charges.shape)
# histogram of pedestal charges (biased extractor) from good and not noisy
# pixels:
qbins = 100
qrange = (-10, 15)
dataq = np.histogram(data_ped_charges[all_good].flatten(),
bins=qbins,
range=qrange,
density=True)
# Find the peak of the pedestal biased charge distribution of real data.
# Use an interpolated version of the histogram, for robustness:
func = interp1d(0.5 * (dataq[1][1:] + dataq[1][:-1]),
dataq[0],
kind='quadratic',
fill_value='extrapolate')
xx = np.linspace(qrange[0], qrange[1], 100 * qbins)
mode_data = xx[np.argmax(func(xx))]
# Event reader for simtel file:
mc_reader = EventSource(input_url=simtel_filename, config=Config(config))
# Obtain the configuration with which the pedestal calculations were
# performed:
ped_config = config['LSTCalibrationCalculator']['PedestalIntegrator']
tel_id = ped_config['tel_id']
# Obtain the (unbiased) extractor used for pedestal calculations:
pedestal_extractor_type = ped_config['charge_product']
pedestal_calibrator = CameraCalibrator(
image_extractor_type=pedestal_extractor_type,
config=Config(ped_config),
subarray=mc_reader.subarray)
# Obtain the (usually biased) extractor used for shower images:
shower_extractor_type = config['image_extractor']
shower_calibrator = CameraCalibrator(
image_extractor_type=shower_extractor_type,
config=Config(config),
subarray=mc_reader.subarray)
# Since these extractors are now for use on MC, we have to apply the pulse
# integration correction (in data that is currently, as of
# lstchain v0.7.5, replaced by an empirical (hard-coded) correction of the
# adc to pe conversion factors )
pedestal_calibrator.image_extractors[
ped_config['charge_product']].apply_integration_correction = True
shower_calibrator.image_extractors[
shower_extractor_type].apply_integration_correction = True
# Pulse integration window width of the (biased) extractor for showers:
shower_extractor_window_width = config[
config['image_extractor']]['window_width']
# Pulse integration window width for the pedestal estimation:
pedestal_extractor_config = ped_config[pedestal_extractor_type]
pedestal_extractor_window_width = pedestal_extractor_config['window_width']
# MC pedestals integrated with the unbiased pedestal extractor
mc_ped_charges = []
# MC pedestals integrated with the biased shower extractor
mc_ped_charges_biased = []
for event in mc_reader:
if tel_id not in event.trigger.tels_with_trigger:
continue
# Extract the signals as we do for pedestals (unbiased fixed window
# extractor):
pedestal_calibrator(event)
charges = event.dl1.tel[tel_id].image
# True number of pe's from Cherenkov photons (to identify noise-only pixels)
true_image = event.simulation.tel[tel_id].true_image
mc_ped_charges.append(charges[true_image == 0])
# Now extract the signal as we would do for shower events (usually
# with a biased extractor, e.g. LocalPeakWindowSum):
shower_calibrator(event)
charges_biased = event.dl1.tel[tel_id].image
mc_ped_charges_biased.append(charges_biased[true_image == 0])
# All pixels behave (for now) in the same way in MC, just put them together
mc_ped_charges = np.concatenate(mc_ped_charges)
mc_ped_charges_biased = np.concatenate(mc_ped_charges_biased)
mcq = np.histogram(mc_ped_charges_biased,
bins=qbins,
range=qrange,
density=True)
# Find the peak of the pedestal biased charge distribution of MC. Use
# an interpolated version of the histogram, for robustness:
func = interp1d(0.5 * (mcq[1][1:] + mcq[1][:-1]),
mcq[0],
kind='quadratic',
fill_value='extrapolate')
xx = np.linspace(qrange[0], qrange[1], 100 * qbins)
mode_mc = xx[np.argmax(func(xx))]
mc_unbiased_std_ped_pe = np.std(mc_ped_charges)
# Find the additional noise (in data w.r.t. MC) for the unbiased extractor,
# and scale it to the width of the window for integration of shower images.
# The idea is that when a strong signal is present, the biased extractor
# will integrate around it, and the additional noise is unbiased because
# it won't modify the integration range.
extra_noise_in_bright_pixels = \
((data_median_std_ped_pe**2 - mc_unbiased_std_ped_pe**2) *
shower_extractor_window_width / pedestal_extractor_window_width)
# Just in case, makes sure we just add noise if the MC noise is smaller
# than the real data's:
extra_noise_in_bright_pixels = max(0., extra_noise_in_bright_pixels)
bias = mode_data - mode_mc
extra_bias_in_dim_pixels = max(bias, 0)
# differences of values to peak charge:
dq = data_ped_charges[all_good].flatten() - mode_data
dqmc = mc_ped_charges_biased - mode_mc
# maximum distance (in pe) from peak, to avoid strong impact of outliers:
maxq = 10
# calculate widening of the noise bump:
added_noise = (np.sum(dq[dq < maxq]**2) / len(dq[dq < maxq]) -
np.sum(dqmc[dqmc < maxq]**2) / len(dqmc[dqmc < maxq]))
added_noise = (max(0, added_noise))**0.5
extra_noise_in_dim_pixels = added_noise
return extra_noise_in_dim_pixels, extra_bias_in_dim_pixels, \
extra_noise_in_bright_pixels
