class FITSIndexWriter(Tool):
name = "FITSIndexWriter"
description = __doc__
example = """
To create DL3 index files with default values:
> lstchain_create_dl3_index_files
-d /path/to/DL3/files/
Or specify some more configurations:
> lstchain_create_dl3_index_files
-d /path/to/DL3/files/
-o /path/to/DL3/index/files
-p dl3*[run_1-run_n]*.fits.gz
--overwrite
"""
input_dl3_dir = traits.Path(
help="Input path of DL3 files",
exists=True,
directory_ok=True,
file_ok=False
).tag(config=True)
file_pattern = traits.Unicode(
help="File pattern to search in the given Path",
default_value="dl3*.fits*"
).tag(config=True)
output_index_path = traits.Path(
help="Output path for the Index files",
exists=True,
directory_ok=True,
file_ok=False,
default_value=None
).tag(config=True)
overwrite = traits.Bool(
help="If True, overwrites existing output file without asking",
default_value=False,
).tag(config=True)
aliases = {
("d", "input-dl3-dir"): "FITSIndexWriter.input_dl3_dir",
("o", "output-index-path"): "FITSIndexWriter.output_index_path",
("p", "file-pattern"): "FITSIndexWriter.file_pattern",
}
flags = {
"overwrite": (
{"FITSIndexWriter": {"overwrite": True}},
"overwrite output files if True",
)
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.file_list = []
self.hdu_index_filename = "hdu-index.fits.gz"
self.obs_index_filename = "obs-index.fits.gz"
def setup(self):
list_files = sorted(self.input_dl3_dir.glob(self.file_pattern))
if list_files == []:
self.log.critical(f"No files found with pattern {self.file_pattern}")
for f in list_files:
self.file_list.append(f.name)
Provenance().add_input_file(f)
if not self.output_index_path:
self.output_index_path = self.input_dl3_dir
self.hdu_index_file = self.output_index_path / self.hdu_index_filename
self.obs_index_file = self.output_index_path / self.obs_index_filename
self.provenance_log = self.output_index_path / (self.name + ".provenance.log")
if self.hdu_index_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.hdu_index_file}")
self.hdu_index_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.hdu_index_file} already exists,"
"use --overwrite to overwrite"
)
if self.obs_index_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.obs_index_file}")
self.obs_index_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.obs_index_file} already exists,"
" use --overwrite to overwrite"
)
self.log.debug("HDU Index file: %s", self.hdu_index_file)
self.log.debug("OBS Index file: %s", self.obs_index_file)
def start(self):
create_hdu_index_hdu(
self.file_list,
self.input_dl3_dir,
self.hdu_index_file,
self.overwrite,
)
create_obs_index_hdu(
self.file_list,
self.input_dl3_dir,
self.obs_index_file,
self.overwrite
)
self.log.debug("HDULists created for the index files")
def finish(self):
Provenance().add_output_file(self.hdu_index_file)
Provenance().add_output_file(self.obs_index_file)
class ImagePlotter(Component):
""" Plotter for camera images """
display = Bool(
True,
help="Display the photoelectron images on-screen as they are produced."
).tag(config=True)
output_path = traits.Path(
directory_ok=False,
help=("Output path for the pdf containing all the images."
" Set to None for no saved output."),
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
"""
Plotter for camera images.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self._current_tel = None
self.c_intensity = None
self.c_peak_time = None
self.cb_intensity = None
self.cb_peak_time = None
self.pdf = None
self.subarray = subarray
self._init_figure()
def _init_figure(self):
self.fig = plt.figure(figsize=(16, 7))
self.ax_intensity = self.fig.add_subplot(1, 2, 1)
self.ax_peak_time = self.fig.add_subplot(1, 2, 2)
if self.output_path:
self.log.info(f"Creating PDF: {self.output_path}")
self.pdf = PdfPages(self.output_path)
def plot(self, event, telid):
image = event.dl1.tel[telid].image
peak_time = event.dl1.tel[telid].peak_time
if self._current_tel != telid:
self._current_tel = telid
self.ax_intensity.cla()
self.ax_peak_time.cla()
# Redraw camera
geom = self.subarray.tel[telid].camera.geometry
self.c_intensity = CameraDisplay(geom, ax=self.ax_intensity)
time_cmap = copy(plt.get_cmap("RdBu_r"))
time_cmap.set_under("gray")
time_cmap.set_over("gray")
self.c_peak_time = CameraDisplay(geom,
ax=self.ax_peak_time,
cmap=time_cmap)
if not self.cb_intensity:
self.c_intensity.add_colorbar(ax=self.ax_intensity,
label="Intensity (p.e.)")
self.cb_intensity = self.c_intensity.colorbar
else:
self.c_intensity.colorbar = self.cb_intensity
self.c_intensity.update(True)
if not self.cb_peak_time:
self.c_peak_time.add_colorbar(ax=self.ax_peak_time,
label="Pulse Time (ns)")
self.cb_peak_time = self.c_peak_time.colorbar
else:
self.c_peak_time.colorbar = self.cb_peak_time
self.c_peak_time.update(True)
self.c_intensity.image = image
self.c_peak_time.image = peak_time
# center around brightes pixel, show 10ns total
t_chargemax = peak_time[image.argmax()]
self.c_peak_time.set_limits_minmax(t_chargemax - 5, t_chargemax + 5)
self.fig.suptitle("Event_index={} Event_id={} Telescope={}".format(
event.count, event.index.event_id, telid))
if self.display:
plt.pause(0.001)
if self.pdf is not None:
self.pdf.savefig(self.fig)
def finish(self):
if self.pdf is not None:
self.log.info("Closing PDF")
self.pdf.close()
class FitIntensityScan(Tool):
"""
Tool that generates a HDF5 file with the results of the fit
of the signal of an intensity scan (filter scan in the case of LST), this is useful to estimate the
quadratic noise term to include in the standard F-factor formula
To be run with
lstchain_fit_intensity_scan --config config.json
"""
name = "FitFilterScan"
description = "Tool to fit an intensity scan"
signal_range = List(
[[1500, 14000], [200, 14000]],
help='Signal range to include in the fit for [HG,LG] (camera median in [ADC])'
).tag(config=True)
gain_channels = List(
[0, 1],
help='Gain channel to process (HG=0, LG=1)'
).tag(config=True)
sub_run = Int(
0,
help='Sub run number to process'
).tag(config=True)
run_list = List(
help='List of runs',
).tag(config=True)
input_dir = traits.Path(
directory_ok=True,
help='directory with the input files',
).tag(config=True)
input_prefix = traits.Unicode(
default_value="calibration",
help='Prefix to select calibration files to fit',
).tag(config=True)
output_path = traits.Path(
directory_ok=False, default_value="filter_scan_fit.h5",
help='Path the output hdf5 file',
).tag(config=True)
plot_path = traits.Path(
directory_ok=False, default_value="filter_scan_fit.pdf",
help='Path to pdf file with check plots',
).tag(config=True)
fit_initialization = List(
[[100.0, 0.001], [6.0, 0.001]],
help='Fit parameters initalization [gain (ADC/pe), B term] for HG and LG'
).tag(config=True)
fractional_variance_error = Float(
0.02,
help='Constant fractional error assumed for the y fit coordinate (variance)'
).tag(config=True)
squared_excess_noise_factor = Float(
1.222,
help='Excess noise factor squared: 1+ Var(gain)/Mean(Gain)**2'
).tag(config=True)
aliases = Dict(dict(
signal_range='FitIntensityScan.signal_range',
input_dir='FitIntensityScan.input_dir',
output_path='FitIntensityScan.output_path',
plot_path='FitIntensityScan.plot_path',
sub_run='FitIntensityScan.sub_run',
gain_channels='FitIntensityScan.gain_channels',
run_list='FitIntensityScan.run_list',
input_prefix='FitIntensityScan.input_prefix',
))
def __init__(self, **kwargs):
super().__init__(**kwargs)
"""
For getting help run:
python calc_camera_calibration.py --help
"""
for chan in self.gain_channels:
if not self.signal_range[chan]:
raise ValueError(f"Trailet signal_range {self.signal_range} inconsistent with"
f"trailet {self.gain_channels}. \n")
self.unusable_pixels = [None, None]
self.signal = [None, None]
self.variance = [None, None]
self.selected_runs = [[], []]
self.fit_parameters = np.zeros((constants.N_GAINS, constants.N_PIXELS, 2))
self.fit_cov_matrix = np.zeros((constants.N_GAINS, constants.N_PIXELS, 4))
self.fit_error = np.zeros((constants.N_GAINS, constants.N_PIXELS))
def setup(self):
ff_data = FlatFieldContainer()
ped_data = PedestalContainer()
calib_data = WaveformCalibrationContainer()
channel = ["HG", "LG"]
# loop on runs and memorize data
try:
for i, run in enumerate(self.run_list):
file_list = sorted(
Path(f"{self.input_dir}").rglob(f'{self.input_prefix}*.Run{run:05d}.{self.sub_run:04d}.h5'))
if len(file_list) == 0:
raise IOError(f"Input file for run {run} do not found. \n")
if len(file_list) > 1:
raise IOError(f"Input file for run {run} is more than one: {file_list} \n")
inp_file = file_list[0]
if os.path.getsize(inp_file) < 100:
raise IOError(f"file size run {run} is too short \n")
if read_calibration_file(inp_file, ff_data, calib_data, ped_data):
self.log.debug(f"Read file {inp_file}")
for chan in self.gain_channels:
# verify that the median signal is inside the asked range
median_charge = np.median(ff_data.charge_median[chan])
if median_charge > self.signal_range[chan][1] or median_charge < self.signal_range[chan][0]:
self.log.debug(
f"{channel[chan]}: skip run {run}, signal out of range {median_charge:6.1f} ADC")
continue
signal = ff_data.charge_median[chan] - ped_data.charge_median[chan]
variance = ff_data.charge_std[chan] ** 2 - ped_data.charge_std[chan] ** 2
if self.signal[chan] is None:
self.signal[chan] = signal
self.variance[chan] = variance
self.unusable_pixels[chan] = calib_data.unusable_pixels[chan]
else:
self.signal[chan] = np.column_stack((self.signal[chan], signal))
self.variance[chan] = np.column_stack((self.variance[chan], variance))
self.unusable_pixels[chan] = np.column_stack(
(self.unusable_pixels[chan], calib_data.unusable_pixels[chan]))
self.selected_runs[chan].append(run)
self.log.info(f"{channel[chan]}: select run {run}, median charge {median_charge:6.1f} ADC\n")
else:
raise IOError(f"--> Problem in reading {run}\n")
# check to have enough selected runs
for chan in self.gain_channels:
if self.signal[chan] is None:
raise IOError(f"--> Zero runs selected for channel {channel[chan]} \n")
if self.signal[chan].size < MIN_N_RUNS * constants.N_PIXELS:
raise IOError(
f"--> Not enough runs selected for channel {channel[chan]}: {int(self.signal[chan].size / constants.N_PIXELS)} runs \n")
except ValueError as e:
self.log.error(e)
def start(self):
'''loop to fit each pixel '''
# only positive parameters
bounds = [0, 200]
funfit = partial(quadratic_fit, f2=self.squared_excess_noise_factor)
for pix in np.arange(constants.N_PIXELS):
if pix % 100 == 0:
self.log.debug(f"Pixel {pix}")
# loop over channel
for chan in self.gain_channels:
# fit parameters initialization
p0 = np.array(self.fit_initialization[chan])
mask = self.unusable_pixels[chan][pix]
sig = np.ma.array(self.signal[chan][pix], mask=mask).compressed()
var = np.ma.array(self.variance[chan][pix], mask=mask).compressed()
# skip the pixel if not enough data
if sig.shape[0] < MIN_N_RUNS:
self.log.debug(
f"Not enough data in pixel {pix} and channel {chan} for the fit ({sig.shape[0]} runs)\n")
self.fit_error[chan, pix] = 1
continue
# we assume a constant fractional error
sigma = self.fractional_variance_error * var
try:
par, par_cov = curve_fit(funfit, sig, var, bounds=bounds, sigma=sigma, p0=p0)
self.fit_parameters[chan, pix] = par
self.fit_cov_matrix[chan, pix] = par_cov.reshape(4)
except Exception as e:
self.log.error(e)
self.log.error(f"Error for pixel {pix} and channel {chan}:\n")
self.log.error(f"signal {sig}\n")
self.log.error(f"variance {var}\n")
self.fit_error[chan, pix] = 1
def finish(self):
"""
write fit results in h5 file and the check-plots in pdf file
"""
gain = np.ma.array(self.fit_parameters.T[0], mask=self.fit_error.T)
quadratic_term = np.ma.array(self.fit_parameters.T[1], mask=self.fit_error.T)
# give to the badly fitted pixel a median value for the B term
median_quadratic_term = np.ma.median(quadratic_term, axis=0)
fill_array = np.ones((constants.N_PIXELS, constants.N_GAINS)) * median_quadratic_term
quadratic_term_corrected = np.ma.filled(quadratic_term, fill_array)
with h5py.File(self.output_path, 'w') as hf:
hf.create_dataset('gain', data=gain.T)
hf.create_dataset('B_term', data=quadratic_term_corrected.T)
hf.create_dataset('covariance_matrix', data=self.fit_cov_matrix)
hf.create_dataset('bad_fit_mask', data=self.fit_error)
# remember the camera median and the variance per run
channel = ["HG", "LG"]
for chan in [0, 1]:
if self.signal[chan] is not None:
hf.create_dataset(f'median_signal_{channel[chan]}', data=np.median(self.signal[chan], axis=0))
hf.create_dataset(f'median_variance_{channel[chan]}', data=np.median(self.variance[chan], axis=0))
hf.create_dataset(f'runs_{channel[chan]}', data=self.selected_runs[chan])
hf.create_dataset('runs', data=self.run_list)
hf.create_dataset('sub_run', data=self.sub_run)
# plot open pdf
with PdfPages(self.plot_path) as pdf:
plt.rc("font", size=15)
for chan in self.gain_channels:
# plot the used runs and their median camera charge
fig = plt.figure((chan + 1), figsize=(8, 20))
fig.suptitle(f"{channel[chan]} channel", fontsize=25)
ax = plt.subplot(2, 1, 1)
ax.grid(True)
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
ax.yaxis.set_major_locator(plt.MultipleLocator(1))
plt.plot(np.median(self.signal[chan], axis=0), self.selected_runs[chan], "o")
plt.xlabel(r'$\mathrm{\overline{Q}-\overline{ped}}$ [ADC]')
plt.ylabel(r'Runs used in the fit')
plt.subplot(2, 1, 2)
camera = load_camera_geometry()
camera = camera.transform_to(EngineeringCameraFrame())
disp = CameraDisplay(camera)
image = self.fit_parameters.T[1].T * 100
mymin = np.median(image[chan]) - 3 * np.std(image[chan])
mymax = np.median(image[chan]) + 3 * np.std(image[chan])
disp.set_limits_minmax(mymin, mymax)
mask = np.where(self.fit_error[chan] == 1)[0]
disp.highlight_pixels(mask, linewidth=2.5, color="green")
disp.image = image[chan]
disp.cmap = plt.cm.coolwarm
plt.title(f"{channel[chan]} Fitted B values [%]")
disp.add_colorbar()
plt.tight_layout()
pdf.savefig()
# plot the fit results and residuals for four arbitrary pixels
fig = plt.figure((chan + 1) * 10, figsize=(11, 22))
fig.suptitle(f"{channel[chan]} channel", fontsize=25)
pad = 0
for pix in [0, 600, 1200, 1800]:
pad += 1
plt.subplot(4, 2, pad)
plt.grid(which='minor')
mask = self.unusable_pixels[chan][pix]
sig = np.ma.array(self.signal[chan][pix], mask=mask).compressed()
var = np.ma.array(self.variance[chan][pix], mask=mask).compressed()
popt = self.fit_parameters[chan, pix]
# plot points
plt.plot(sig, var, 'o', color="C0")
# plot fit
min_x = min(1000, np.min(sig) * 0.9)
max_x = max(10000, np.max(sig) * 1.1)
x = np.arange(np.min(sig), np.max(sig))
plt.plot(x, quadratic_fit(x, *popt), '--', color="C1",
label=f'Pixel {pix}:\ng={popt[0]:5.2f} [ADC/pe] , B={popt[1]:5.3f}')
plt.xlim(min_x, max_x)
plt.xlabel('Q-ped [ADC]')
plt.ylabel(r'$\mathrm{\sigma_Q^2-\sigma_{ped}^2}$ [$ADC^2$]')
plt.xscale('log')
plt.yscale('log')
plt.legend()
# plot residuals
pad += 1
plt.subplot(4, 2, pad)
plt.grid(which='both', axis='both')
popt = self.fit_parameters[chan, pix]
plt.plot(sig, (quadratic_fit(sig, *popt) - var) / var * 100, 'o', color="C0")
plt.xlim(min_x, max_x)
plt.xscale('log')
plt.ylabel('fit residuals %')
plt.xlabel('Q-ped [ADC]')
plt.hlines(0, 0, np.max(sig), linestyle='dashed', color="black")
plt.tight_layout()
pdf.savefig()
class IRFFITSWriter(Tool):
name = "IRFFITSWriter"
description = __doc__
example = """
To generate IRFs from MC gamma only, using default cuts/binning:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--overwrite
Or to generate all 4 IRFs, using default cuts/binning:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-p /path/to/DL2_MC_proton_file.h5
-e /path/to/DL2_MC_electron_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
Or use a config file for cuts and binning information:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--config /path/to/config.json
Or pass the selection cuts from command-line:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--global-gh-cut 0.9
--global-theta-cut 0.2
--irf-obs-time 50
Or use energy-dependent cuts based on a gamma efficiency:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--energy-dependent-gh
--energy-dependent-theta
--gh-efficiency 0.95
--theta-containment 0.68
Or generate source-dependent IRFs
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like
--global-gh-cut 0.9
--global-alpha-cut 10
--source-dep
"""
input_gamma_dl2 = traits.Path(help="Input MC gamma DL2 file",
allow_none=True,
exists=True,
directory_ok=False,
file_ok=True).tag(config=True)
input_proton_dl2 = traits.Path(help="Input MC proton DL2 file",
allow_none=True,
exists=True,
directory_ok=False,
file_ok=True).tag(config=True)
input_electron_dl2 = traits.Path(help="Input MC electron DL2 file",
allow_none=True,
exists=True,
directory_ok=False,
file_ok=True).tag(config=True)
output_irf_file = traits.Path(
help="IRF output file",
allow_none=True,
directory_ok=False,
file_ok=True,
default_value="./irf.fits.gz",
).tag(config=True)
irf_obs_time = traits.Float(
help="Observation time for IRF in hours",
default_value=50,
).tag(config=True)
point_like = traits.Bool(
help="True for point_like IRF, False for Full Enclosure",
default_value=False,
).tag(config=True)
energy_dependent_gh = traits.Bool(
help="True for applying energy-dependent gammaness cuts",
default_value=False,
).tag(config=True)
energy_dependent_theta = traits.Bool(
help="True for applying energy-dependent theta cuts",
default_value=False,
).tag(config=True)
energy_dependent_alpha = traits.Bool(
help="True for applying energy-dependent alpha cuts",
default_value=False,
).tag(config=True)
overwrite = traits.Bool(
help="If True, overwrites existing output file without asking",
default_value=False,
).tag(config=True)
source_dep = traits.Bool(
help="True for source-dependent analysis",
default_value=False,
).tag(config=True)
classes = [EventSelector, DL3Cuts, DataBinning]
aliases = {
("g", "input-gamma-dl2"): "IRFFITSWriter.input_gamma_dl2",
("p", "input-proton-dl2"): "IRFFITSWriter.input_proton_dl2",
("e", "input-electron-dl2"): "IRFFITSWriter.input_electron_dl2",
("o", "output-irf-file"): "IRFFITSWriter.output_irf_file",
"irf-obs-time": "IRFFITSWriter.irf_obs_time",
"global-gh-cut": "DL3Cuts.global_gh_cut",
"gh-efficiency": "DL3Cuts.gh_efficiency",
"theta-containment": "DL3Cuts.theta_containment",
"global-theta-cut": "DL3Cuts.global_theta_cut",
"alpha-containment": "DL3Cuts.alpha_containment",
"global-alpha-cut": "DL3Cuts.global_alpha_cut",
"allowed-tels": "DL3Cuts.allowed_tels",
"overwrite": "IRFFITSWriter.overwrite",
}
flags = {
"point-like": (
{
"IRFFITSWriter": {
"point_like": True
}
},
"Point like IRFs will be produced, otherwise Full Enclosure",
),
"overwrite": (
{
"IRFFITSWriter": {
"overwrite": True
}
},
"overwrites output file",
),
"source-dep": (
{
"IRFFITSWriter": {
"source_dep": True
}
},
"Source-dependent analysis will be performed",
),
"energy-dependent-gh": (
{
"IRFFITSWriter": {
"energy_dependent_gh": True
}
},
"Uses energy-dependent cuts for gammaness",
),
"energy-dependent-theta": (
{
"IRFFITSWriter": {
"energy_dependent_theta": True
}
},
"Uses energy-dependent cuts for theta",
),
"energy-dependent-alpha": (
{
"IRFFITSWriter": {
"energy_dependent_alpha": True
}
},
"Uses energy-dependent cuts for alpha",
),
}
def setup(self):
if self.output_irf_file.absolute().exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.output_irf_file}")
self.output_irf_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_irf_file} already exists,"
" use --overwrite to overwrite")
filename = self.output_irf_file.name
if not (filename.endswith('.fits') or filename.endswith('.fits.gz')):
raise ValueError(
f"{filename} is not a correct compressed FITS file name"
"(use .fits or .fits.gz).")
if self.input_proton_dl2 and self.input_electron_dl2 is not Undefined:
self.only_gamma_irf = False
else:
self.only_gamma_irf = True
self.event_sel = EventSelector(parent=self)
self.cuts = DL3Cuts(parent=self)
self.data_bin = DataBinning(parent=self)
self.mc_particle = {
"gamma": {
"file": self.input_gamma_dl2,
"target_spectrum": CRAB_MAGIC_JHEAP2015,
},
}
Provenance().add_input_file(self.input_gamma_dl2)
self.t_obs = self.irf_obs_time * u.hour
# Read and update MC information
if not self.only_gamma_irf:
self.mc_particle["proton"] = {
"file": self.input_proton_dl2,
"target_spectrum": IRFDOC_PROTON_SPECTRUM,
}
self.mc_particle["electron"] = {
"file": self.input_electron_dl2,
"target_spectrum": IRFDOC_ELECTRON_SPECTRUM,
}
Provenance().add_input_file(self.input_proton_dl2)
Provenance().add_input_file(self.input_electron_dl2)
self.provenance_log = self.output_irf_file.parent / (self.name +
".provenance.log")
def start(self):
for particle_type, p in self.mc_particle.items():
self.log.info(f"Simulated {particle_type.title()} Events:")
p["events"], p["simulation_info"] = read_mc_dl2_to_QTable(
p["file"])
p["mc_type"] = check_mc_type(p["file"])
self.log.debug(
f"Simulated {p['mc_type']} {particle_type.title()} Events:")
# Calculating event weights for Background IRF
if particle_type != "gamma":
p["simulated_spectrum"] = PowerLaw.from_simulation(
p["simulation_info"], self.t_obs)
p["events"]["weight"] = calculate_event_weights(
p["events"]["true_energy"],
p["target_spectrum"],
p["simulated_spectrum"],
)
if not self.source_dep:
for prefix in ("true", "reco"):
k = f"{prefix}_source_fov_offset"
p["events"][k] = calculate_source_fov_offset(p["events"],
prefix=prefix)
# calculate theta / distance between reco and assumed source position
p["events"]["theta"] = calculate_theta(
p["events"],
assumed_source_az=p["events"]["true_az"],
assumed_source_alt=p["events"]["true_alt"],
)
else:
# Alpha cut is applied for source-dependent analysis.
# To adapt source-dependent analysis to pyirf codes,
# true position is set as reco position for survived events
# after alpha cut
p["events"][
"true_source_fov_offset"] = calculate_source_fov_offset(
p["events"], prefix="true")
p["events"]["reco_source_fov_offset"] = p["events"][
"true_source_fov_offset"]
self.log.debug(p["simulation_info"])
gammas = self.mc_particle["gamma"]["events"]
# Binning of parameters used in IRFs
true_energy_bins = self.data_bin.true_energy_bins()
reco_energy_bins = self.data_bin.reco_energy_bins()
migration_bins = self.data_bin.energy_migration_bins()
source_offset_bins = self.data_bin.source_offset_bins()
gammas = self.event_sel.filter_cut(gammas)
gammas = self.cuts.allowed_tels_filter(gammas)
if self.energy_dependent_gh:
self.gh_cuts_gamma = self.cuts.energy_dependent_gh_cuts(
gammas, reco_energy_bins)
gammas = self.cuts.apply_energy_dependent_gh_cuts(
gammas, self.gh_cuts_gamma)
self.log.info(
f"Using gamma efficiency of {self.cuts.gh_efficiency}")
else:
gammas = self.cuts.apply_global_gh_cut(gammas)
self.log.info("Using a global gammaness cut of "
f"{self.cuts.global_gh_cut}")
if self.point_like:
if not self.source_dep:
if self.energy_dependent_theta:
self.theta_cuts = self.cuts.energy_dependent_theta_cuts(
gammas,
reco_energy_bins,
)
gammas = self.cuts.apply_energy_dependent_theta_cuts(
gammas, self.theta_cuts)
self.log.info("Using a containment region for theta of "
f"{self.cuts.theta_containment}")
else:
gammas = self.cuts.apply_global_theta_cut(gammas)
self.log.info(
"Using a global Theta cut of "
f"{self.cuts.global_theta_cut} for point-like IRF")
else:
if self.energy_dependent_alpha:
self.alpha_cuts = self.cuts.energy_dependent_alpha_cuts(
gammas,
reco_energy_bins,
)
gammas = self.cuts.apply_energy_dependent_alpha_cuts(
gammas, self.alpha_cuts)
self.log.info("Using a containment region for alpha of "
f"{self.cuts.alpha_containment} %")
else:
gammas = self.cuts.apply_global_alpha_cut(gammas)
self.log.info(
'Using a global Alpha cut of '
f'{self.cuts.global_alpha_cut} for point like IRF')
if self.mc_particle["gamma"]["mc_type"] in [
"point_like", "ring_wobble"
]:
mean_fov_offset = round(
gammas["true_source_fov_offset"].mean().to_value(), 1)
fov_offset_bins = [mean_fov_offset - 0.1, mean_fov_offset + 0.1
] * u.deg
self.log.info('Single offset for point like gamma MC')
else:
fov_offset_bins = self.data_bin.fov_offset_bins()
self.log.info('Multiple offset for diffuse gamma MC')
if self.energy_dependent_theta:
fov_offset_bins = [
round(gammas["true_source_fov_offset"].min().to_value(),
1),
round(gammas["true_source_fov_offset"].max().to_value(), 1)
] * u.deg
self.log.info("For RAD MAX, the full FoV is used")
if not self.only_gamma_irf:
background = table.vstack([
self.mc_particle["proton"]["events"],
self.mc_particle["electron"]["events"]
])
if self.energy_dependent_gh:
background = self.cuts.apply_energy_dependent_gh_cuts(
background, self.gh_cuts_gamma)
else:
background = self.cuts.apply_global_gh_cut(background)
background = self.event_sel.filter_cut(background)
background = self.cuts.allowed_tels_filter(background)
background_offset_bins = self.data_bin.bkg_fov_offset_bins()
# For a global gh/theta cut, only a header value is added.
# For energy-dependent cuts, along with GADF specified RAD_MAX HDU,
# a new HDU is created, GH_CUTS which is based on RAD_MAX table
# NOTE: The GH_CUTS HDU is just for provenance and is not supported
# by GADF or used by any Science Tools
extra_headers = {
"TELESCOP": "CTA-N",
"INSTRUME": "LST-" + " ".join(map(str, self.cuts.allowed_tels)),
"FOVALIGN": "RADEC",
}
if self.point_like:
self.log.info("Generating point_like IRF HDUs")
else:
self.log.info("Generating Full-Enclosure IRF HDUs")
# Updating the HDU headers with the gammaness and theta cuts/efficiency
if not self.energy_dependent_gh:
extra_headers["GH_CUT"] = self.cuts.global_gh_cut
else:
extra_headers["GH_EFF"] = (self.cuts.gh_efficiency,
"gamma/hadron efficiency")
if self.point_like:
if not self.source_dep:
if self.energy_dependent_theta:
extra_headers["TH_CONT"] = (
self.cuts.theta_containment,
"Theta containment region in percentage")
else:
extra_headers["RAD_MAX"] = (self.cuts.global_theta_cut,
'deg')
else:
if self.energy_dependent_alpha:
extra_headers["AL_CONT"] = (
self.cuts.alpha_containment,
"Alpha containment region in percentage")
else:
extra_headers["AL_CUT"] = (self.cuts.global_alpha_cut,
'deg')
# Write HDUs
self.hdus = [
fits.PrimaryHDU(),
]
with np.errstate(invalid="ignore", divide="ignore"):
if self.mc_particle["gamma"]["mc_type"] in [
"point_like", "ring_wobble"
]:
self.effective_area = effective_area_per_energy(
gammas,
self.mc_particle["gamma"]["simulation_info"],
true_energy_bins,
)
self.hdus.append(
create_aeff2d_hdu(
# add one dimension for single FOV offset
self.effective_area[..., np.newaxis],
true_energy_bins,
fov_offset_bins,
point_like=self.point_like,
extname="EFFECTIVE AREA",
**extra_headers,
))
else:
self.effective_area = effective_area_per_energy_and_fov(
gammas,
self.mc_particle["gamma"]["simulation_info"],
true_energy_bins,
fov_offset_bins,
)
self.hdus.append(
create_aeff2d_hdu(
self.effective_area,
true_energy_bins,
fov_offset_bins,
point_like=self.point_like,
extname="EFFECTIVE AREA",
**extra_headers,
))
self.log.info("Effective Area HDU created")
self.edisp = energy_dispersion(
gammas,
true_energy_bins,
fov_offset_bins,
migration_bins,
)
self.hdus.append(
create_energy_dispersion_hdu(
self.edisp,
true_energy_bins,
migration_bins,
fov_offset_bins,
point_like=self.point_like,
extname="ENERGY DISPERSION",
**extra_headers,
))
self.log.info("Energy Dispersion HDU created")
if not self.only_gamma_irf:
self.background = background_2d(
background,
reco_energy_bins=reco_energy_bins,
fov_offset_bins=background_offset_bins,
t_obs=self.t_obs,
)
self.hdus.append(
create_background_2d_hdu(
self.background.T,
reco_energy_bins,
background_offset_bins,
extname="BACKGROUND",
**extra_headers,
))
self.log.info("Background HDU created")
if not self.point_like:
self.psf = psf_table(
gammas,
true_energy_bins,
fov_offset_bins=fov_offset_bins,
source_offset_bins=source_offset_bins,
)
self.hdus.append(
create_psf_table_hdu(
self.psf,
true_energy_bins,
source_offset_bins,
fov_offset_bins,
extname="PSF",
**extra_headers,
))
self.log.info("PSF HDU created")
if self.energy_dependent_gh:
# Create a separate temporary header
gh_header = fits.Header()
gh_header["CREATOR"] = f"lstchain v{__version__}"
gh_header["DATE"] = Time.now().utc.iso
for k, v in extra_headers.items():
gh_header[k] = v
self.hdus.append(
fits.BinTableHDU(self.gh_cuts_gamma,
header=gh_header,
name="GH_CUTS"))
self.log.info("GH CUTS HDU added")
if self.energy_dependent_theta and self.point_like:
if not self.source_dep:
self.hdus.append(
create_rad_max_hdu(self.theta_cuts["cut"][:, np.newaxis],
reco_energy_bins, fov_offset_bins,
**extra_headers))
self.log.info("RAD MAX HDU added")
if self.energy_dependent_alpha and self.source_dep:
# Create a separate temporary header
alpha_header = fits.Header()
alpha_header["CREATOR"] = f"lstchain v{__version__}"
alpha_header["DATE"] = Time.now().utc.iso
for k, v in extra_headers.items():
alpha_header[k] = v
self.hdus.append(
fits.BinTableHDU(self.alpha_cuts,
header=gh_header,
name="AL_CUTS"))
self.log.info("ALPHA CUTS HDU added")
def finish(self):
fits.HDUList(self.hdus).writeto(self.output_irf_file,
overwrite=self.overwrite)
Provenance().add_output_file(self.output_irf_file)
class DataReductionFITSWriter(Tool):
name = "DataReductionFITSWriter"
description = __doc__
example = """
To generate DL3 file from an observed data DL2 file, using default cuts:
> lstchain_create_dl3_file
-d /path/to/DL2_data_file.h5
-o /path/to/DL3/file/
--input-irf /path/to/irf.fits.gz
--source-name Crab
--source-ra 83.633deg
--source-dec 22.01deg
Or use a config file for the cuts:
> lstchain_create_dl3_file
-d /path/to/DL2_data_file.h5
-o /path/to/DL3/file/
--input-irf /path/to/irf.fits.gz
--source-name Crab
--source-ra 83.633deg
--source-dec 22.01deg
--overwrite
--config /path/to/config.json
Or pass the selection cuts from command-line:
> lstchain_create_dl3_file
-d /path/to/DL2_data_file.h5
-o /path/to/DL3/file/
--input-irf /path/to/irf.fits.gz
--source-name Crab
--source-ra 83.633deg
--source-dec 22.01deg
--global-gh-cut 0.9
--overwrite
Or generate source-dependent DL3 files
> lstchain_create_dl3_file
-d /path/to/DL2_data_file.h5
-o /path/to/DL3/file/
--input-irf /path/to/irf.fits.gz
--source-name Crab
--source-dep
--overwrite
"""
input_dl2 = traits.Path(help="Input data DL2 file",
exists=True,
directory_ok=False,
file_ok=True).tag(config=True)
output_dl3_path = traits.Path(help="DL3 output filedir",
directory_ok=True,
file_ok=False).tag(config=True)
input_irf = traits.Path(
help="Compressed FITS file of IRFs",
exists=True,
directory_ok=False,
file_ok=True,
).tag(config=True)
source_name = traits.Unicode(help="Name of Source").tag(config=True)
source_ra = traits.Unicode(help="RA position of the source").tag(
config=True)
source_dec = traits.Unicode(help="DEC position of the source").tag(
config=True)
overwrite = traits.Bool(
help="If True, overwrites existing output file without asking",
default_value=False,
).tag(config=True)
source_dep = traits.Bool(
help="If True, source-dependent analysis will be performed.",
default_value=False,
).tag(config=True)
classes = [EventSelector, DL3Cuts]
aliases = {
("d", "input-dl2"): "DataReductionFITSWriter.input_dl2",
("o", "output-dl3-path"): "DataReductionFITSWriter.output_dl3_path",
"input-irf": "DataReductionFITSWriter.input_irf",
"global-gh-cut": "DL3Cuts.global_gh_cut",
"source-name": "DataReductionFITSWriter.source_name",
"source-ra": "DataReductionFITSWriter.source_ra",
"source-dec": "DataReductionFITSWriter.source_dec",
}
flags = {
"overwrite": (
{
"DataReductionFITSWriter": {
"overwrite": True
}
},
"overwrite output file if True",
),
"source-dep": (
{
"DataReductionFITSWriter": {
"source_dep": True
}
},
"source-dependent analysis if True",
),
}
def setup(self):
self.filename_dl3 = dl2_to_dl3_filename(self.input_dl2)
self.provenance_log = self.output_dl3_path / (self.name +
".provenance.log")
Provenance().add_input_file(self.input_dl2)
self.event_sel = EventSelector(parent=self)
self.cuts = DL3Cuts(parent=self)
self.output_file = self.output_dl3_path.absolute() / self.filename_dl3
if self.output_file.exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.output_file}")
self.output_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_file} already exists,"
" use --overwrite to overwrite")
if not (self.source_ra or self.source_dec):
self.source_pos = SkyCoord.from_name(self.source_name)
elif bool(self.source_ra) != bool(self.source_dec):
raise ToolConfigurationError(
"Either provide both RA and DEC values for the source or none")
else:
self.source_pos = SkyCoord(ra=self.source_ra, dec=self.source_dec)
self.log.debug(f"Output DL3 file: {self.output_file}")
try:
with fits.open(self.input_irf) as hdul:
self.use_energy_dependent_cuts = (
"GH_CUT" not in hdul["EFFECTIVE AREA"].header)
except:
raise ToolConfigurationError(
f"{self.input_irf} does not have EFFECTIVE AREA HDU, "
" to check for global cut information in the Header value")
def apply_srcindep_gh_cut(self):
''' apply gammaness cut '''
self.data = self.event_sel.filter_cut(self.data)
if self.use_energy_dependent_cuts:
self.energy_dependent_gh_cuts = QTable.read(self.input_irf,
hdu="GH_CUTS")
self.data = self.cuts.apply_energy_dependent_gh_cuts(
self.data, self.energy_dependent_gh_cuts)
self.log.info("Using gamma efficiency of "
f"{self.energy_dependent_gh_cuts.meta['GH_EFF']}")
else:
with fits.open(self.input_irf) as hdul:
self.cuts.global_gh_cut = hdul[1].header["GH_CUT"]
self.data = self.cuts.apply_global_gh_cut(self.data)
self.log.info(f"Using global G/H cut of {self.cuts.global_gh_cut}")
def apply_srcdep_gh_alpha_cut(self):
''' apply gammaness and alpha cut for source-dependent analysis '''
srcdep_assumed_positions = get_srcdep_assumed_positions(self.input_dl2)
for i, srcdep_pos in enumerate(srcdep_assumed_positions):
data_temp = read_data_dl2_to_QTable(self.input_dl2,
srcdep_pos=srcdep_pos)
data_temp = self.event_sel.filter_cut(data_temp)
if self.use_energy_dependent_cuts:
self.energy_dependent_gh_cuts = QTable.read(self.input_irf,
hdu="GH_CUTS")
data_temp = self.cuts.apply_energy_dependent_gh_cuts(
data_temp, self.energy_dependent_gh_cuts)
else:
with fits.open(self.input_irf) as hdul:
self.cuts.global_gh_cut = hdul[1].header["GH_CUT"]
data_temp = self.cuts.apply_global_gh_cut(data_temp)
with fits.open(self.input_irf) as hdul:
self.cuts.global_alpha_cut = hdul[1].header["AL_CUT"]
data_temp = self.cuts.apply_global_alpha_cut(data_temp)
# set expected source positions as reco positions
set_expected_pos_to_reco_altaz(data_temp)
if i == 0:
self.data = data_temp
else:
self.data = vstack([self.data, data_temp])
def start(self):
if not self.source_dep:
self.data = read_data_dl2_to_QTable(self.input_dl2)
else:
self.data = read_data_dl2_to_QTable(self.input_dl2, 'on')
self.effective_time, self.elapsed_time = get_effective_time(self.data)
self.run_number = run_info_from_filename(self.input_dl2)[1]
if not self.source_dep:
self.apply_srcindep_gh_cut()
else:
self.apply_srcdep_gh_alpha_cut()
self.data = add_icrs_position_params(self.data, self.source_pos)
self.log.info("Generating event list")
self.events, self.gti, self.pointing = create_event_list(
data=self.data,
run_number=self.run_number,
source_name=self.source_name,
source_pos=self.source_pos,
effective_time=self.effective_time.value,
elapsed_time=self.elapsed_time.value,
)
self.hdulist = fits.HDUList(
[fits.PrimaryHDU(), self.events, self.gti, self.pointing])
irf = fits.open(self.input_irf)
self.log.info("Adding IRF HDUs")
for irf_hdu in irf[1:]:
self.hdulist.append(irf_hdu)
def finish(self):
self.hdulist.writeto(self.output_file, overwrite=self.overwrite)
Provenance().add_output_file(self.output_file)
class MuonAnalysis(Tool):
"""
Detect and extract muon ring parameters, and write the muon ring and
intensity parameters to an output table.
The resulting output can be read e.g. using for example
`pandas.read_hdf(filename, 'dl1/event/telescope/parameters/muon')`
"""
name = "ctapipe-reconstruct-muons"
description = traits.Unicode(__doc__)
output = traits.Path(directory_ok=False,
help="HDF5 output file name").tag(config=True)
completeness_threshold = traits.FloatTelescopeParameter(
default_value=30.0,
help="Threshold for calculating the ``ring_completeness``").tag(
config=True)
ratio_width = traits.FloatTelescopeParameter(
default_value=1.5,
help=("Ring width for intensity ratio"
" computation as multiple of pixel diameter"),
).tag(config=True)
overwrite = traits.Bool(
default_value=False,
help="If true, overwrite outputfile without asking").tag(config=True)
min_pixels = traits.IntTelescopeParameter(
help=("Minimum number of pixels after cleaning and ring finding"
"required to process an event"),
default_value=100,
).tag(config=True)
pedestal = traits.FloatTelescopeParameter(
help="Pedestal noise rms", default_value=1.1).tag(config=True)
classes = [
CameraCalibrator,
TailcutsImageCleaner,
EventSource,
MuonRingFitter,
MuonIntensityFitter,
]
aliases = {
"i": "EventSource.input_url",
"input": "EventSource.input_url",
"o": "MuonAnalysis.output",
"output": "MuonAnalysis.output",
"max-events": "EventSource.max_events",
"allowed-tels": "EventSource.allowed_tels",
}
flags = {
"overwrite": ({
"MuonAnalysis": {
"overwrite": True
}
}, "overwrite output file")
}
def setup(self):
if self.output is None:
raise ToolConfigurationError(
"You need to provide an --output file")
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
"Outputfile {self.output} already exists, use `--overwrite` to overwrite"
)
self.source = EventSource(parent=self)
subarray = self.source.subarray
self.calib = CameraCalibrator(subarray=subarray, parent=self)
self.ring_fitter = MuonRingFitter(parent=self)
self.intensity_fitter = MuonIntensityFitter(subarray=subarray,
parent=self)
self.cleaning = TailcutsImageCleaner(parent=self, subarray=subarray)
self.writer = HDF5TableWriter(self.output,
"",
add_prefix=True,
parent=self,
mode="w")
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
"min_pixels", "pedestal", "ratio_width",
"completeness_threshold"
]:
getattr(self, p).attach_subarray(self.source.subarray)
def start(self):
for event in tqdm(self.source, desc="Processing events: "):
self.process_array_event(event)
def process_array_event(self, event):
self.calib(event)
for tel_id, dl1 in event.dl1.tel.items():
self.process_telescope_event(event.index, tel_id, dl1)
self.writer.write("sim/event/subarray/shower",
[event.index, event.simulation.shower])
def process_telescope_event(self, event_index, tel_id, dl1):
event_id = event_index.event_id
if self.source.subarray.tel[tel_id].optics.num_mirrors != 1:
self.log.warn(f"Skipping non-single mirror telescope {tel_id}"
" set --allowed_tels to get rid of this warning")
return
self.log.debug(f"Processing event {event_id}, telescope {tel_id}")
image = dl1.image
if dl1.image_mask is None:
dl1.image_mask = self.cleaning(tel_id, image)
if np.count_nonzero(dl1.image_mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f"Skipping event {event_id}-{tel_id}:"
f" has less then {self.min_pixels.tel[tel_id]} pixels after cleaning"
)
return
x, y = self.get_pixel_coords(tel_id)
# iterative ring fit.
# First use cleaning pixels, then only pixels close to the ring
# three iterations seems to be enough for most rings
mask = dl1.image_mask
for i in range(3):
ring = self.ring_fitter(x, y, image, mask)
dist = np.sqrt((x - ring.center_x)**2 + (y - ring.center_y)**2)
mask = np.abs(dist - ring.radius) / ring.radius < 0.4
if np.count_nonzero(mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f"Skipping event {event_id}-{tel_id}:"
f" Less then {self.min_pixels.tel[tel_id]} pixels on ring")
return
if np.isnan(
[ring.radius.value, ring.center_x.value,
ring.center_y.value]).any():
self.log.debug(
f"Skipping event {event_id}-{tel_id}: Ring fit did not succeed"
)
return
parameters = self.calculate_muon_parameters(tel_id, image,
dl1.image_mask, ring)
# intensity_fitter does not support a mask yet, set ignored pixels to 0
image[~mask] = 0
result = self.intensity_fitter(
tel_id,
ring.center_x,
ring.center_y,
ring.radius,
image,
pedestal=self.pedestal.tel[tel_id],
)
self.log.info(f"Muon fit: r={ring.radius:.2f}"
f", width={result.width:.4f}"
f", efficiency={result.optical_efficiency:.2%}")
tel_event_index = TelEventIndexContainer(**event_index, tel_id=tel_id)
self.writer.write(
"dl1/event/telescope/parameters/muons",
[tel_event_index, ring, parameters, result],
)
def calculate_muon_parameters(self, tel_id, image, clean_mask, ring):
fov_radius = self.get_fov(tel_id)
x, y = self.get_pixel_coords(tel_id)
# add ring containment, not filled in fit
containment = ring_containment(ring.radius, ring.center_x,
ring.center_y, fov_radius)
completeness = ring_completeness(
x,
y,
image,
ring.radius,
ring.center_x,
ring.center_y,
threshold=self.completeness_threshold.tel[tel_id],
)
pixel_width = self.get_pixel_width(tel_id)
intensity_ratio = intensity_ratio_inside_ring(
x[clean_mask],
y[clean_mask],
image[clean_mask],
ring.radius,
ring.center_x,
ring.center_y,
width=self.ratio_width.tel[tel_id] * pixel_width,
)
mse = mean_squared_error(
x[clean_mask],
y[clean_mask],
image[clean_mask],
ring.radius,
ring.center_x,
ring.center_y,
)
return MuonParametersContainer(
containment=containment,
completeness=completeness,
intensity_ratio=intensity_ratio,
mean_squared_error=mse,
)
def get_fov(self, tel_id):
"""Guesstimate fov radius for telescope with id `tel_id`"""
# memoize fov calculation
if tel_id not in self.field_of_view:
cam = self.source.subarray.tel[tel_id].camera.geometry
border = cam.get_border_pixel_mask()
x, y = self.get_pixel_coords(tel_id)
self.field_of_view[tel_id] = np.sqrt(x[border]**2 +
y[border]**2).mean()
return self.field_of_view[tel_id]
def get_pixel_width(self, tel_id):
"""Guesstimate fov radius for telescope with id `tel_id`"""
# memoize fov calculation
if tel_id not in self.pixel_widths:
x, y = self.get_pixel_coords(tel_id)
self.pixel_widths[tel_id] = CameraGeometry.guess_pixel_width(x, y)
return self.pixel_widths[tel_id]
def get_pixel_coords(self, tel_id):
"""Get pixel coords in telescope frame for telescope with id `tel_id`"""
# memoize transformation
if tel_id not in self.pixels_in_tel_frame:
telescope = self.source.subarray.tel[tel_id]
cam = telescope.camera.geometry
camera_frame = CameraFrame(
focal_length=telescope.optics.equivalent_focal_length,
rotation=cam.cam_rotation,
)
cam_coords = SkyCoord(x=cam.pix_x, y=cam.pix_y, frame=camera_frame)
tel_coord = cam_coords.transform_to(TelescopeFrame())
self.pixels_in_tel_frame[tel_id] = tel_coord
coords = self.pixels_in_tel_frame[tel_id]
return coords.fov_lon, coords.fov_lat
def finish(self):
Provenance().add_output_file(self.output,
role="muon_efficiency_parameters")
self.writer.close()
class IRFFITSWriter(Tool):
name = "IRFFITSWriter"
description = __doc__
example = """
To generate IRFs from MC gamma only, using default cuts/binning:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--overwrite
Or to generate all 4 IRFs, using default cuts/binning:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-p /path/to/DL2_MC_proton_file.h5
-e /path/to/DL2_MC_electron_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
Or use a config file for cuts and binning information:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--config /path/to/config.json
Or pass the selection cuts from command-line:
> lstchain_create_irf_files
-g /path/to/DL2_MC_gamma_file.h5
-o /path/to/irf.fits.gz
--point-like (Only for point_like IRFs)
--fixed-gh-cut 0.9
--fixed-theta-cut 0.2
--irf-obs-time 50
"""
input_gamma_dl2 = traits.Path(
help="Input MC gamma DL2 file",
exists=True,
directory_ok=False,
file_ok=True
).tag(config=True)
input_proton_dl2 = traits.Path(
help="Input MC proton DL2 file",
exists=True,
directory_ok=False,
file_ok=True
).tag(config=True)
input_electron_dl2 = traits.Path(
help="Input MC electron DL2 file",
exists=True,
directory_ok=False,
file_ok=True
).tag(config=True)
output_irf_file = traits.Path(
help="IRF output file",
directory_ok=False,
file_ok=True,
default_value="./irf.fits.gz",
).tag(config=True)
irf_obs_time = traits.Float(
help="Observation time for IRF in hours",
default_value=50,
).tag(config=True)
point_like = traits.Bool(
help="True for point_like IRF, False for Full Enclosure",
default_value=False,
).tag(config=True)
overwrite = traits.Bool(
help="If True, overwrites existing output file without asking",
default_value=False,
).tag(config=True)
classes = [EventSelector, DL3FixedCuts, DataBinning]
aliases = {
("g", "input-gamma-dl2"): "IRFFITSWriter.input_gamma_dl2",
("p", "input-proton-dl2"): "IRFFITSWriter.input_proton_dl2",
("e", "input-electron-dl2"): "IRFFITSWriter.input_electron_dl2",
("o", "output-irf-file"): "IRFFITSWriter.output_irf_file",
"irf-obs-time": "IRFFITSWriter.irf_obs_time",
"fixed-gh-cut": "DL3FixedCuts.fixed_gh_cut",
"fixed-theta-cut": "DL3FixedCuts.fixed_theta_cut",
"allowed-tels": "DL3FixedCuts.allowed_tels",
"overwrite": "IRFFITSWriter.overwrite",
}
flags = {
"point-like": (
{"IRFFITSWriter": {"point_like": True}},
"Point like IRFs will be produced, otherwise Full Enclosure",
),
"overwrite": (
{"IRFFITSWriter": {"overwrite": True}},
"overwrites output file",
)
}
def setup(self):
if self.output_irf_file.absolute().exists():
if self.overwrite:
self.log.warning(f"Overwriting {self.output_irf_file}")
self.output_irf_file.unlink()
else:
raise ToolConfigurationError(
f"Output file {self.output_irf_file} already exists,"
" use --overwrite to overwrite"
)
filename = self.output_irf_file.name
if not (filename.endswith('.fits') or filename.endswith('.fits.gz')):
raise ValueError("f{filename} is not a correct compressed FITS file name (use .fits or .fits.gz).")
if self.input_proton_dl2 and self.input_electron_dl2 is not None:
self.only_gamma_irf = False
else:
self.only_gamma_irf = True
self.event_sel = EventSelector(parent=self)
self.fixed_cuts = DL3FixedCuts(parent=self)
self.data_bin = DataBinning(parent=self)
self.mc_particle = {
"gamma": {
"file": str(self.input_gamma_dl2),
"target_spectrum": CRAB_MAGIC_JHEAP2015,
},
}
Provenance().add_input_file(self.input_gamma_dl2)
self.t_obs = self.irf_obs_time * u.hour
# Read and update MC information
if not self.only_gamma_irf:
self.mc_particle["proton"] = {
"file": str(self.input_proton_dl2),
"target_spectrum": IRFDOC_PROTON_SPECTRUM,
}
self.mc_particle["electron"] = {
"file": str(self.input_electron_dl2),
"target_spectrum": IRFDOC_ELECTRON_SPECTRUM,
}
Provenance().add_input_file(self.input_proton_dl2)
Provenance().add_input_file(self.input_electron_dl2)
self.provenance_log = self.output_irf_file.parent / (
self.name + ".provenance.log"
)
def start(self):
for particle_type, p in self.mc_particle.items():
self.log.info(f"Simulated {particle_type.title()} Events:")
p["events"], p["simulation_info"] = read_mc_dl2_to_QTable(p["file"])
if p["simulation_info"].viewcone.value == 0.0:
p["mc_type"] = "point_like"
else:
p["mc_type"] = "diffuse"
self.log.debug(f"Simulated {p['mc_type']} {particle_type.title()} Events:")
# Calculating event weights for Background IRF
if particle_type != "gamma":
p["simulated_spectrum"] = PowerLaw.from_simulation(
p["simulation_info"], self.t_obs
)
p["events"]["weight"] = calculate_event_weights(
p["events"]["true_energy"],
p["target_spectrum"],
p["simulated_spectrum"],
)
for prefix in ("true", "reco"):
k = f"{prefix}_source_fov_offset"
p["events"][k] = calculate_source_fov_offset(p["events"], prefix=prefix)
# calculate theta / distance between reco and assumed source position
p["events"]["theta"] = calculate_theta(
p["events"],
assumed_source_az=p["events"]["true_az"],
assumed_source_alt=p["events"]["true_alt"],
)
self.log.debug(p["simulation_info"])
gammas = self.mc_particle["gamma"]["events"]
self.log.info(f"Using fixed G/H cut of {self.fixed_cuts.fixed_gh_cut}")
gammas = self.event_sel.filter_cut(gammas)
gammas = self.fixed_cuts.allowed_tels_filter(gammas)
gammas = self.fixed_cuts.gh_cut(gammas)
if self.point_like:
gammas = self.fixed_cuts.theta_cut(gammas)
self.log.info('Theta cuts applied for point like IRF')
# Binning of parameters used in IRFs
true_energy_bins = self.data_bin.true_energy_bins()
reco_energy_bins = self.data_bin.reco_energy_bins()
migration_bins = self.data_bin.energy_migration_bins()
source_offset_bins = self.data_bin.source_offset_bins()
if self.mc_particle["gamma"]["mc_type"] == "point_like":
mean_fov_offset = round(gammas["true_source_fov_offset"].mean().to_value(), 1)
fov_offset_bins = [mean_fov_offset - 0.1, mean_fov_offset + 0.1] * u.deg
self.log.info('Single offset for point like gamma MC')
else:
fov_offset_bins = self.data_bin.fov_offset_bins()
self.log.info('Multiple offset for diffuse gamma MC')
if not self.only_gamma_irf:
background = table.vstack(
[
self.mc_particle["proton"]["events"],
self.mc_particle["electron"]["events"],
]
)
background = self.event_sel.filter_cut(background)
background = self.fixed_cuts.allowed_tels_filter(background)
background = self.fixed_cuts.gh_cut(background)
background_offset_bins = self.data_bin.bkg_fov_offset_bins()
# For a fixed gh/theta cut, only a header value is added.
# For energy dependent cuts, a new HDU should be created
# GH_CUT and FOV_CUT are temporary non-standard header data
extra_headers = {
"TELESCOP": "CTA-N",
"INSTRUME": "LST-" + " ".join(map(str, self.fixed_cuts.allowed_tels)),
"FOVALIGN": "RADEC",
"GH_CUT": self.fixed_cuts.fixed_gh_cut,
}
if self.point_like:
self.log.info("Generating point_like IRF HDUs")
extra_headers["RAD_MAX"] = str(self.fixed_cuts.fixed_theta_cut * u.deg)
else:
self.log.info("Generating Full-Enclosure IRF HDUs")
# Write HDUs
self.hdus = [fits.PrimaryHDU(), ]
with np.errstate(invalid="ignore", divide="ignore"):
if self.mc_particle["gamma"]["mc_type"] == "point_like":
self.effective_area = effective_area_per_energy(
gammas,
self.mc_particle["gamma"]["simulation_info"],
true_energy_bins,
)
self.hdus.append(
create_aeff2d_hdu(
# add one dimension for single FOV offset
self.effective_area[..., np.newaxis],
true_energy_bins,
fov_offset_bins,
point_like=self.point_like,
extname="EFFECTIVE AREA",
**extra_headers,
)
)
else:
self.effective_area = effective_area_per_energy_and_fov(
gammas,
self.mc_particle["gamma"]["simulation_info"],
true_energy_bins,
fov_offset_bins,
)
self.hdus.append(
create_aeff2d_hdu(
self.effective_area,
true_energy_bins,
fov_offset_bins,
point_like=self.point_like,
extname="EFFECTIVE AREA",
**extra_headers,
)
)
self.log.info("Effective Area HDU created")
self.edisp = energy_dispersion(
gammas,
true_energy_bins,
fov_offset_bins,
migration_bins,
)
self.hdus.append(
create_energy_dispersion_hdu(
self.edisp,
true_energy_bins,
migration_bins,
fov_offset_bins,
point_like=self.point_like,
extname="ENERGY DISPERSION",
**extra_headers,
)
)
self.log.info("Energy Dispersion HDU created")
if not self.only_gamma_irf:
self.background = background_2d(
background,
reco_energy_bins=reco_energy_bins,
fov_offset_bins=background_offset_bins,
t_obs=self.t_obs,
)
self.hdus.append(
create_background_2d_hdu(
self.background.T,
reco_energy_bins,
background_offset_bins,
extname="BACKGROUND",
**extra_headers,
)
)
self.log.info("Background HDU created")
if not self.point_like:
self.psf = psf_table(
gammas,
true_energy_bins,
fov_offset_bins=fov_offset_bins,
source_offset_bins=source_offset_bins,
)
self.hdus.append(
create_psf_table_hdu(
self.psf,
true_energy_bins,
source_offset_bins,
fov_offset_bins,
extname="PSF",
**extra_headers,
)
)
self.log.info("PSF HDU created")
def finish(self):
fits.HDUList(self.hdus).writeto(self.output_irf_file, overwrite=self.overwrite)
Provenance().add_output_file(self.output_irf_file)
class MuonAnalysis(Tool):
"""
Detect and extract muon ring parameters, and write the muon ring and
intensity parameters to an output table.
The resulting output can be read e.g. using for example
`pandas.read_hdf(filename, 'dl1/event/telescope/parameters/muon')`
"""
name = 'ctapipe-reconstruct-muons'
description = traits.Unicode(__doc__)
output = traits.Path(directory_ok=False,
help='HDF5 output file name').tag(config=True)
completeness_threshold = traits.FloatTelescopeParameter(
default_value=30.0,
help='Threshold for calculating the ``ring_completeness``',
).tag(config=True)
ratio_width = traits.FloatTelescopeParameter(
default_value=1.5,
help=('Ring width for intensity ratio'
' computation as multiple of pixel diameter')).tag(config=True)
overwrite = traits.Bool(
default_value=False,
help='If true, overwrite outputfile without asking').tag(config=True)
min_pixels = traits.IntTelescopeParameter(
help=('Minimum number of pixels after cleaning and ring finding'
'required to process an event'),
default_value=100,
).tag(config=True)
pedestal = traits.FloatTelescopeParameter(
help='Pedestal noise rms',
default_value=1.1,
).tag(config=True)
extractor_name = traits.create_class_enum_trait(
ImageExtractor,
default_value='GlobalPeakWindowSum',
).tag(config=True)
classes = [
CameraCalibrator,
TailcutsImageCleaner,
EventSource,
MuonRingFitter,
MuonIntensityFitter,
] + traits.classes_with_traits(ImageExtractor)
aliases = {
'i': 'EventSource.input_url',
'input': 'EventSource.input_url',
'o': 'MuonAnalysis.output',
'output': 'MuonAnalysis.output',
'max-events': 'EventSource.max_events',
'allowed-tels': 'EventSource.allowed_tels',
}
flags = {
'overwrite': ({
'MuonAnalysis': {
'overwrite': True
}
}, 'overwrite output file')
}
def setup(self):
if self.output is None:
raise ToolConfigurationError(
'You need to provide an --output file')
if self.output.exists() and not self.overwrite:
raise ToolConfigurationError(
'Outputfile {self.output} already exists, use `--overwrite` to overwrite'
)
self.source = self.add_component(EventSource.from_config(parent=self))
self.extractor = self.add_component(
ImageExtractor.from_name(self.extractor_name,
parent=self,
subarray=self.source.subarray))
self.calib = self.add_component(
CameraCalibrator(
subarray=self.source.subarray,
parent=self,
image_extractor=self.extractor,
))
self.ring_fitter = self.add_component(MuonRingFitter(parent=self, ))
self.intensity_fitter = self.add_component(
MuonIntensityFitter(
subarray=self.source.subarray,
parent=self,
))
self.cleaning = self.add_component(
TailcutsImageCleaner(
parent=self,
subarray=self.source.subarray,
))
self.writer = self.add_component(
HDF5TableWriter(
self.output,
"",
add_prefix=True,
parent=self,
mode='w',
))
self.pixels_in_tel_frame = {}
self.field_of_view = {}
self.pixel_widths = {}
for p in [
'min_pixels', 'pedestal', 'ratio_width',
'completeness_threshold'
]:
getattr(self, p).attach_subarray(self.source.subarray)
def start(self):
for event in tqdm(self.source, desc='Processing events: '):
self.process_array_event(event)
def process_array_event(self, event):
self.calib(event)
for tel_id, dl1 in event.dl1.tel.items():
self.process_telescope_event(event.index, tel_id, dl1)
self.writer.write('sim/event/subarray/shower', [event.index, event.mc])
def process_telescope_event(self, event_index, tel_id, dl1):
event_id = event_index.event_id
if self.source.subarray.tel[tel_id].optics.num_mirrors != 1:
self.log.warn(f'Skipping non-single mirror telescope {tel_id}'
' set --allowed_tels to get rid of this warning')
return
self.log.debug(f'Processing event {event_id}, telescope {tel_id}')
image = dl1.image
clean_mask = self.cleaning(tel_id, image)
if np.count_nonzero(clean_mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f'Skipping event {event_id}-{tel_id}:'
f' has less then {self.min_pixels.tel[tel_id]} pixels after cleaning'
)
return
x, y = self.get_pixel_coords(tel_id)
# iterative ring fit.
# First use cleaning pixels, then only pixels close to the ring
# three iterations seems to be enough for most rings
mask = clean_mask
for i in range(3):
ring = self.ring_fitter(x, y, image, mask)
dist = np.sqrt((x - ring.center_x)**2 + (y - ring.center_y)**2)
mask = np.abs(dist - ring.radius) / ring.radius < 0.4
if np.count_nonzero(mask) <= self.min_pixels.tel[tel_id]:
self.log.debug(
f'Skipping event {event_id}-{tel_id}:'
f' Less then {self.min_pixels.tel[tel_id]} pixels on ring')
return
if np.isnan(
[ring.radius.value, ring.center_x.value,
ring.center_y.value]).any():
self.log.debug(
f'Skipping event {event_id}-{tel_id}: Ring fit did not succeed'
)
return
parameters = self.calculate_muon_parameters(tel_id, image, clean_mask,
ring)
# intensity_fitter does not support a mask yet, set ignored pixels to 0
image[~mask] = 0
result = self.intensity_fitter(
tel_id,
ring.center_x,
ring.center_y,
ring.radius,
image,
pedestal=self.pedestal.tel[tel_id],
)
self.log.info(
f'Muon fit: r={ring.radius:.2f}'
f', width={result.width:.4f}'
f', efficiency={result.optical_efficiency:.2%}', )
tel_event_index = TelEventIndexContainer(
**event_index,
tel_id=tel_id,
)
self.writer.write('dl1/event/telescope/parameters/muons',
[tel_event_index, ring, parameters, result])
def calculate_muon_parameters(self, tel_id, image, clean_mask, ring):
fov_radius = self.get_fov(tel_id)
x, y = self.get_pixel_coords(tel_id)
# add ring containment, not filled in fit
containment = ring_containment(
ring.radius,
ring.center_x,
ring.center_y,
fov_radius,
)
completeness = ring_completeness(
x,
y,
image,
ring.radius,
ring.center_x,
ring.center_y,
threshold=self.completeness_threshold.tel[tel_id],
)
pixel_width = self.get_pixel_width(tel_id)
intensity_ratio = intensity_ratio_inside_ring(
x[clean_mask],
y[clean_mask],
image[clean_mask],
ring.radius,
ring.center_x,
ring.center_y,
width=self.ratio_width.tel[tel_id] * pixel_width,
)
mse = mean_squared_error(x[clean_mask], y[clean_mask],
image[clean_mask], ring.radius, ring.center_x,
ring.center_y)
return MuonParametersContainer(
containment=containment,
completeness=completeness,
intensity_ratio=intensity_ratio,
mean_squared_error=mse,
)
def get_fov(self, tel_id):
'''Guesstimate fov radius for telescope with id `tel_id`'''
# memoize fov calculation
if tel_id not in self.field_of_view:
cam = self.source.subarray.tel[tel_id].camera.geometry
border = cam.get_border_pixel_mask()
x, y = self.get_pixel_coords(tel_id)
self.field_of_view[tel_id] = np.sqrt(x[border]**2 +
y[border]**2).mean()
return self.field_of_view[tel_id]
def get_pixel_width(self, tel_id):
'''Guesstimate fov radius for telescope with id `tel_id`'''
# memoize fov calculation
if tel_id not in self.pixel_widths:
x, y = self.get_pixel_coords(tel_id)
self.pixel_widths[tel_id] = CameraGeometry.guess_pixel_width(x, y)
return self.pixel_widths[tel_id]
def get_pixel_coords(self, tel_id):
'''Get pixel coords in telescope frame for telescope with id `tel_id`'''
# memoize transformation
if tel_id not in self.pixels_in_tel_frame:
telescope = self.source.subarray.tel[tel_id]
cam = telescope.camera.geometry
camera_frame = CameraFrame(
focal_length=telescope.optics.equivalent_focal_length,
rotation=cam.cam_rotation,
)
cam_coords = SkyCoord(x=cam.pix_x, y=cam.pix_y, frame=camera_frame)
tel_coord = cam_coords.transform_to(TelescopeFrame())
self.pixels_in_tel_frame[tel_id] = tel_coord
coords = self.pixels_in_tel_frame[tel_id]
return coords.fov_lon, coords.fov_lat
def finish(self):
Provenance().add_output_file(
self.output,
role='muon_efficiency_parameters',
)
self.writer.close()
class ImagePlotter(Component):
""" Plotter for camera images """
display = Bool(
True,
help="Display the photoelectron images on-screen as they are produced."
).tag(config=True)
output_path = traits.Path(
directory_ok=False,
help=("Output path for the pdf containing all the images."
" Set to None for no saved output."),
).tag(config=True)
def __init__(self, subarray, config=None, parent=None, **kwargs):
"""
Plotter for camera images.
Parameters
----------
config : traitlets.loader.Config
Configuration specified by config file or cmdline arguments.
Used to set traitlet values.
Set to None if no configuration to pass.
tool : ctapipe.core.Tool
Tool executable that is calling this component.
Passes the correct logger to the component.
Set to None if no Tool to pass.
kwargs
"""
super().__init__(config=config, parent=parent, **kwargs)
self._current_tel = None
self.c_intensity = None
self.c_peak_time = None
self.cb_intensity = None
self.cb_peak_time = None
self.pdf = None
self.subarray = subarray
self._init_figure()
def _init_figure(self):
self.fig = plt.figure(figsize=(16, 7))
self.ax_intensity = self.fig.add_subplot(1, 2, 1)
self.ax_peak_time = self.fig.add_subplot(1, 2, 2)
if self.output_path:
self.log.info(f"Creating PDF: {self.output_path}")
self.pdf = PdfPages(self.output_path)
def plot(self, event, telid):
image = event.dl1.tel[telid].image
peak_time = event.dl1.tel[telid].peak_time
print("plot", image.shape, peak_time.shape)
if self._current_tel != telid:
self._current_tel = telid
self.ax_intensity.cla()
self.ax_peak_time.cla()
# Redraw camera
geom = self.subarray.tel[telid].camera.geometry
self.c_intensity = CameraDisplay(geom, ax=self.ax_intensity)
self.c_peak_time = CameraDisplay(geom, ax=self.ax_peak_time)
if (peak_time != 0.0).all():
tmaxmin = event.dl0.tel[telid].waveform.shape[1]
t_chargemax = peak_time[image.argmax()]
cmap_time = colors.LinearSegmentedColormap.from_list(
"cmap_t",
[
(0 / tmaxmin, "darkgreen"),
(0.6 * t_chargemax / tmaxmin, "green"),
(t_chargemax / tmaxmin, "yellow"),
(1.4 * t_chargemax / tmaxmin, "blue"),
(1, "darkblue"),
],
)
self.c_peak_time.pixels.set_cmap(cmap_time)
if not self.cb_intensity:
self.c_intensity.add_colorbar(ax=self.ax_intensity,
label="Intensity (p.e.)")
self.cb_intensity = self.c_intensity.colorbar
else:
self.c_intensity.colorbar = self.cb_intensity
self.c_intensity.update(True)
if not self.cb_peak_time:
self.c_peak_time.add_colorbar(ax=self.ax_peak_time,
label="Pulse Time (ns)")
self.cb_peak_time = self.c_peak_time.colorbar
else:
self.c_peak_time.colorbar = self.cb_peak_time
self.c_peak_time.update(True)
self.c_intensity.image = image
if peak_time is not None:
self.c_peak_time.image = peak_time
self.fig.suptitle("Event_index={} Event_id={} Telescope={}".format(
event.count, event.index.event_id, telid))
if self.display:
plt.pause(0.001)
if self.pdf is not None:
self.pdf.savefig(self.fig)
def finish(self):
if self.pdf is not None:
self.log.info("Closing PDF")
self.pdf.close()
