def define_flux_crab_above_energy(emin=1 * u.TeV, emax=10 * u.TeV):
crab = CrabSpectrum('meyer').model
crabMAGIC = LogParabola(amplitude=3.23e-11 * u.Unit('cm-2 s-1 TeV-1'), reference=1 * u.TeV, alpha=2.47, beta=0.24)
crab_flux_above_1TeV = crabMAGIC.integral(emin=emin, emax=emax)
crab_flux_above_1TeV_model = crab.integral(emin=emin, emax=emax)
return crab_flux_above_1TeV, crab_flux_above_1TeV_model
def plot_meyer_2010():
model_meyer_ref = CrabSpectrum("meyer").model
model_meyer_ref.plot(
[10 * u.GeV, 100 * u.TeV],
energy_power=2,
flux_unit="erg-1 cm-2 s-1",
ls=":",
lw=2.2,
color="#555555",
label="Meyer et al. (2010)",
)
def fill_derived_spectral_info(self):
"""
Fill derived spectral info computed from basic parameters
"""
from gammapy.spectrum import CrabSpectrum
data = self.data
# total errors
data['spec_norm_err_tot'] = np.hypot(data['spec_norm_err'],
data['spec_norm_err_sys'])
data['spec_index_err_tot'] = np.hypot(data['spec_index_err'],
data['spec_index_err_sys'])
spec_model = self._get_spec_model(data)
# Integral flux above 1 TeV
emin, emax = 1, 1E6 # TeV
flux_above_1TeV = spec_model.integral(emin, emax)
data['spec_flux_above_1TeV'] = flux_above_1TeV.n
data['spec_flux_above_1TeV_err'] = flux_above_1TeV.s
# Integral flux above 1 TeV in crab units
crab = CrabSpectrum('meyer').model
emin, emax = 1, 1E6 # TeV
flux_above_1TeV = spec_model.integral(emin, emax)
flux_above_1TeV_crab = crab.integral(emin * u.TeV, emax * u.TeV)
flux_cu = (flux_above_1TeV / flux_above_1TeV_crab.value) * 100
data['spec_flux_above_1TeV_crab'] = flux_cu.n
data['spec_flux_above_1TeV_crab_err'] = flux_cu.s
# Integral flux above erange_min
emin, emax = data['spec_erange_min'], 1E6 # TeV
try:
flux_above_erange_min = spec_model.integral(emin, emax)
data['spec_flux_above_erange_min'] = flux_above_erange_min.n
data['spec_flux_above_erange_min_err'] = flux_above_erange_min.s
except ValueError:
data['spec_flux_above_erange_min'] = NA.fill_value['number']
data['spec_flux_above_erange_min_err'] = NA.fill_value['number']
# Energy flux between 1 TeV and 10 TeV
emin, emax = 1, 10 # TeV
energy_flux = spec_model.energy_flux(emin, emax)
data['spec_energy_flux_1TeV_10TeV'] = energy_flux.n
data['spec_energy_flux_1TeV_10TeV_err'] = energy_flux.s
norm_1TeV = spec_model(1) # TeV
data['spec_norm_1TeV'] = norm_1TeV.n
data['spec_norm_1TeV_err'] = norm_1TeV.s
def get_logn_logs(self, quantity, variant):
crab_1_10 = CrabSpectrum().model.integral(1 * u.TeV, 10 *
u.TeV).to('cm-2 s-1').value
bins = np.logspace(-6.1, 0.7, 100)
hists = []
for component in self.get_components():
table = component['table']
points = table['flux_1_10'] / crab_1_10
hist = Histogram.from_points(bins=bins, points=points)
if quantity == 'n':
pass
elif quantity == 'f':
hist.vals = hist.vals * hist.bin_centers
else:
raise ValueError('Invalid quantity: {}'.format(quantity))
if variant == 'diff':
pass
elif variant == 'int':
hist.vals = np.cumsum(hist.vals[::-1])[::-1]
else:
raise ValueError('Invalid variant: {}'.format(variant))
hists.append(hist)
return HistogramStack(hists)
def get_logn_logs(self, quantity, variant, tags):
#print('ere')
#bins = np.logspace(-3.1, 0.7, 1000)
bins = np.logspace(-0.5, 2.1, 1000)
crab_1_10 = CrabSpectrum().model.integral(1 * u.TeV, 10 * u.TeV).to('cm-2 s-1').value
hists = []
hists_skip = []
for component in self.get_components(tags=tags):
table = component['table']
tag = component['tag']
if tag in {'pwn', 'composite'}:
fluxes = []
fluxes_skip = []
for row in table:
flux = row['int_flux_above_1TeV_cu']
fluxes.append(flux)
if (row['skip'] == 0):
fluxes_skip.append(flux)
if ( tag == 'gammacat'):
fluxes = []
fluxes_skip = []
for row in table:
flux = (row['flux_1_10']/crab_1_10)*100
fluxes.append(flux)
fluxes_skip.append(flux)
hist = Histogram.from_points(bins=bins, points=fluxes)
hist_skip = Histogram.from_points(bins=bins, points=fluxes_skip)
if quantity == 'n':
pass
elif quantity == 'f':
hist.vals = hist.vals * hist.bin_centers
hist_skip.vals = hist_skip.vals * hist_skip.bin_centers
else:
raise ValueError('Invalid quantity: {}'.format(quantity))
if variant == 'diff':
pass
elif variant == 'int':
hist.vals = np.cumsum(hist.vals[::-1])[::-1]
hist_skip.vals = np.cumsum(hist_skip.vals[::-1])[::-1]
else:
raise ValueError('Invalid variant: {}'.format(variant))
hists.append(hist)
hists_skip.append(hist_skip)
return HistogramStack(hists_skip), HistogramStack(hists)
def plot_crab():
"""Plot Crab pulsar and nebula SED."""
log.info("Executing plot_crab ...")
fig, ax = plt.subplots()
# Plot flux points
for component in ["pulsar", "nebula"]:
table = Table.read("data/other/crab_mwl.fits.gz")
table = table[table["component"] == component]
x = table["energy"].data
y = table["energy_flux"].data
yerr_lo = table["energy_flux_err_lo"].data
yerr_hi = table["energy_flux_err_hi"].data
ax.errorbar(x, y, yerr=(yerr_lo, yerr_hi), fmt="o", label=component)
# Plot SED model
energy = np.logspace(2, 8, 100) * u.MeV
crab = CrabSpectrum(reference="meyer")
flux = crab.model(energy)
energy_flux = (energy**2 * flux).to("erg cm^-2 s^-1")
ax.plot(energy.value, energy_flux.value, label="Meyer (2010) model", lw=3)
ax.set_xlim((3e-1, 3e8))
ax.set_ylim((3e-12, 3e-8))
ax.set_xlabel("Energy (MeV)")
ax.set_ylabel("E^2 dN/dE (erg cm^-2 s^-1)")
fig.legend(loc="upper center", ncol=3)
ax.grid()
ax.loglog()
path = Path("results/figures")
path.mkdir(parents=True, exist_ok=True)
filename = "results/figures/crab_mwl.png"
log.info(f"Writing {filename}")
fig.savefig(filename)
def main():
# Read arguments
parser = argparse.ArgumentParser(description='Make performance files')
parser.add_argument('--config_file', type=str, required=True, help='')
parser.add_argument(
'--obs_time',
type=str,
required=True,
help=
'Observation time, should be given as a string, value and astropy unit separated by an empty space'
)
mode_group = parser.add_mutually_exclusive_group()
mode_group.add_argument('--wave',
dest="mode",
action='store_const',
const="wave",
default="tail",
help="if set, use wavelet cleaning")
mode_group.add_argument(
'--tail',
dest="mode",
action='store_const',
const="tail",
help="if set, use tail cleaning, otherwise wavelets")
args = parser.parse_args()
# Read configuration file
cfg = load_config(args.config_file)
# Add obs. time in configuration file
str_obs_time = args.obs_time.split()
cfg['analysis']['obs_time'] = {
'value': float(str_obs_time[0]),
'unit': str(str_obs_time[-1])
}
# Create output directory if necessary
outdir = os.path.join(
cfg['general']['outdir'], 'irf_{}_ThSq_{}_Time{:.2f}{}'.format(
args.mode, cfg['analysis']['thsq_opt']['type'],
cfg['analysis']['obs_time']['value'],
cfg['analysis']['obs_time']['unit']))
if not os.path.exists(outdir):
os.makedirs(outdir)
indir = cfg['general']['indir']
template_input_file = cfg['general']['template_input_file']
# Load data
particles = ['gamma', 'electron', 'proton']
evt_dict = dict() # Contain DL2 file for each type of particle
for particle in particles:
# template looks like dl2_{}_{}_merged.h5
infile = os.path.join(indir,
template_input_file.format(args.mode, particle))
evt_dict[particle] = pd.read_hdf(infile, key='reco_events')
# Apply offset cut to proton and electron
for particle in ['electron', 'proton']:
# print('Initial stat: {} {}'.format(len(evt_dict[particle]), particle))
evt_dict[particle] = evt_dict[particle].query('offset <= {}'.format(
cfg['particle_information'][particle]['offset_cut']))
# Add required data in configuration file for future computation
for particle in particles:
cfg['particle_information'][particle]['n_files'] = \
len(np.unique(evt_dict[particle]['obs_id']))
cfg['particle_information'][particle]['n_simulated'] = \
cfg['particle_information'][particle]['n_files'] * cfg['particle_information'][particle]['n_events_per_file']
# Define model for the particles
model_dict = {
'gamma': CrabSpectrum('hegra').model,
'proton': cosmic_ray_flux,
'electron': cosmic_ray_flux
}
# Reco energy binning
cfg_binning = cfg['analysis']['ereco_binning']
ereco = np.logspace(np.log10(cfg_binning['emin']),
np.log10(cfg_binning['emax']),
cfg_binning['nbin'] + 1) * u.TeV
# Handle theta square cut optimisation
# (compute 68 % containment radius PSF if necessary)
thsq_opt_type = cfg['analysis']['thsq_opt']['type']
if thsq_opt_type in 'fixed':
thsq_values = np.array([cfg['analysis']['thsq_opt']['value']]) * u.deg
print('Using fixed theta cut: {}'.format(thsq_values))
elif thsq_opt_type in 'opti':
thsq_values = np.arange(0.05, 0.40, 0.01) * u.deg
print('Optimising theta cut for: {}'.format(thsq_values))
elif thsq_opt_type in 'r68':
print('Using R68% theta cut')
print('Computing...')
cfg_binning = cfg['analysis']['ereco_binning']
ereco = np.logspace(np.log10(cfg_binning['emin']),
np.log10(cfg_binning['emax']),
cfg_binning['nbin'] + 1) * u.TeV
radius = 68
thsq_values = list()
for ibin in range(len(ereco) - 1):
emin = ereco[ibin]
emax = ereco[ibin + 1]
energy_query = 'reco_energy > {} and reco_energy <= {}'.format(
emin.value, emax.value)
data = evt_dict['gamma'].query(energy_query).copy()
min_stat = 0
if len(data) <= min_stat:
print(' ==> Not enough statistics:')
print('To be handled...')
thsq_values.append(0.3)
continue
# import sys
# sys.exit()
psf = np.percentile(data['offset'], radius)
psf_err = psf / np.sqrt(len(data))
thsq_values.append(psf)
thsq_values = np.array(thsq_values) * u.deg
# Set 0.05 as a lower value
idx = np.where(thsq_values.value < 0.05)
thsq_values[idx] = 0.05 * u.deg
print('Using theta cut: {}'.format(thsq_values))
# Cuts optimisation
print('### Finding best cuts...')
cut_optimiser = CutsOptimisation(config=cfg,
evt_dict=evt_dict,
verbose_level=0)
# Weight events
print('- Weighting events...')
cut_optimiser.weight_events(
model_dict=model_dict,
colname_mc_energy=cfg['column_definition']['mc_energy'])
# Find best cutoff to reach best sensitivity
print('- Estimating cutoffs...')
cut_optimiser.find_best_cutoff(energy_values=ereco,
angular_values=thsq_values)
# Save results and auxiliary data for diagnostic
print('- Saving results to disk...')
cut_optimiser.write_results(outdir,
'{}.fits'.format(
cfg['general']['output_table_name']),
format='fits')
# Cuts diagnostic
print('### Building cut diagnostics...')
cut_diagnostic = CutsDiagnostic(config=cfg, indir=outdir)
cut_diagnostic.plot_optimisation_summary()
cut_diagnostic.plot_diagnostics()
# Apply cuts and save data
print('### Applying cuts to data...')
cut_applicator = CutsApplicator(config=cfg,
evt_dict=evt_dict,
outdir=outdir)
cut_applicator.apply_cuts()
# Irf Maker
print('### Building IRF...')
irf_maker = IrfMaker(config=cfg, evt_dict=evt_dict, outdir=outdir)
irf_maker.build_irf()
# Sensitivity maker
print('### Estimating sensitivity...')
sensitivity_maker = SensitivityMaker(config=cfg, outdir=outdir)
sensitivity_maker.load_irf()
sensitivity_maker.estimate_sensitivity()
def main(args):
paths = {}
paths['gamma'] = args.dl2_gamma_filename
paths['proton'] = args.dl2_proton_filename
paths['electron'] = args.dl2_electron_filename
# Read configuration file
cfg = load_config(args.config_file)
# cfg = configuration()
cfg['analysis']['obs_time'] = {}
cfg['analysis']['obs_time']['unit'] = u.h
cfg['analysis']['obs_time']['value'] = args.obs_time
cfg['general']['outdir'] = args.outdir
# Create output directory if necessary
outdir = os.path.join(
cfg['general']['outdir'],
'irf_ThSq_{}_Time{:.2f}{}'.format(cfg['analysis']['thsq_opt']['type'],
cfg['analysis']['obs_time']['value'],
cfg['analysis']['obs_time']['unit']))
if not os.path.exists(outdir):
os.makedirs(outdir)
# Load data
particles = ['gamma', 'electron', 'proton']
evt_dict = dict() # Contain DL2 file for each type of particle
for particle in particles:
infile = paths[particle]
evt_dict[particle] = read_and_update_dl2(infile)
cfg = get_simu_info(infile, particle, config=cfg)
# Apply offset cut to proton and electron
for particle in ['electron', 'proton']:
evt_dict[particle] = evt_dict[particle].query('offset <= {}'.format(
cfg['particle_information'][particle]['offset_cut']))
# Add required data in configuration file for future computation
for particle in particles:
# cfg['particle_information'][particle]['n_files'] = \
# len(np.unique(evt_dict[particle]['obs_id']))
cfg['particle_information'][particle]['n_simulated'] = \
cfg['particle_information'][particle]['n_files'] * cfg['particle_information'][particle][
'n_events_per_file']
# Define model for the particles
model_dict = {
'gamma': CrabSpectrum('hegra').model,
'proton': cosmic_ray_flux,
'electron': cosmic_ray_flux
}
# Reco energy binning
cfg_binning = cfg['analysis']['ereco_binning']
# ereco = np.logspace(np.log10(cfg_binning['emin']),
# np.log10(cfg_binning['emax']),
# cfg_binning['nbin'] + 1) * u.TeV
ereco = ctaplot.ana.irf_cta().E_bin * u.TeV
# Handle theta square cut optimisation
# (compute 68 % containment radius PSF if necessary)
thsq_opt_type = cfg['analysis']['thsq_opt']['type']
print(thsq_opt_type)
# if thsq_opt_type in 'fixed':
# thsq_values = np.array([cfg['analysis']['thsq_opt']['value']]) * u.deg
# print('Using fixed theta cut: {}'.format(thsq_values))
# elif thsq_opt_type in 'opti':
# thsq_values = np.arange(0.05, 0.40, 0.01) * u.deg
# print('Optimising theta cut for: {}'.format(thsq_values))
if thsq_opt_type != 'r68':
raise ValueError("only r68 supported at the moment")
elif thsq_opt_type in 'r68':
print('Using R68% theta cut')
print('Computing...')
cfg_binning = cfg['analysis']['ereco_binning']
ereco = np.logspace(np.log10(cfg_binning['emin']),
np.log10(cfg_binning['emax']),
cfg_binning['nbin'] + 1) * u.TeV
ereco = ctaplot.ana.irf_cta().E_bin * u.TeV
radius = 68
thsq_values = list()
for ibin in range(len(ereco) - 1):
emin = ereco[ibin]
emax = ereco[ibin + 1]
energy_query = 'reco_energy > {} and reco_energy <= {}'.format(
emin.value, emax.value)
data = evt_dict['gamma'].query(energy_query).copy()
min_stat = 0
if len(data) <= min_stat:
print(' ==> Not enough statistics:')
print('To be handled...')
thsq_values.append(0.3)
continue
psf = np.percentile(data['offset'], radius)
thsq_values.append(psf)
thsq_values = np.array(thsq_values) * u.deg
# Set 0.05 as a lower value
idx = np.where(thsq_values.value < 0.05)
thsq_values[idx] = 0.05 * u.deg
print('Using theta cut: {}'.format(thsq_values))
# Cuts optimisation
print('### Finding best cuts...')
cut_optimiser = CutsOptimisation(config=cfg,
evt_dict=evt_dict,
verbose_level=0)
# Weight events
print('- Weighting events...')
cut_optimiser.weight_events(
model_dict=model_dict,
colname_mc_energy=cfg['column_definition']['mc_energy'])
# Find best cutoff to reach best sensitivity
print('- Estimating cutoffs...')
cut_optimiser.find_best_cutoff(energy_values=ereco,
angular_values=thsq_values)
# Save results and auxiliary data for diagnostic
print('- Saving results to disk...')
cut_optimiser.write_results(outdir,
'{}.fits'.format(
cfg['general']['output_table_name']),
format='fits')
# Cuts diagnostic
print('### Building cut diagnostics...')
cut_diagnostic = CutsDiagnostic(config=cfg, indir=outdir)
cut_diagnostic.plot_optimisation_summary()
cut_diagnostic.plot_diagnostics()
# Apply cuts and save data
print('### Applying cuts to data...')
cut_applicator = CutsApplicator(config=cfg,
evt_dict=evt_dict,
outdir=outdir)
cut_applicator.apply_cuts()
# Irf Maker
print('### Building IRF...')
irf_maker = IrfMaker(config=cfg, evt_dict=evt_dict, outdir=outdir)
irf_maker.build_irf()
# Sensitivity maker
print('### Estimating sensitivity...')
sensitivity_maker = SensitivityMaker(config=cfg, outdir=outdir)
sensitivity_maker.load_irf()
sensitivity_maker.estimate_sensitivity()
analysis.run(optimize_opts={"print_level": 1})
# ## Results
#
# Let's look at the results, and also compare with a previously published Crab nebula spectrum for reference.
# In[ ]:
print(analysis.fit.result[0])
# In[ ]:
opts = {
"energy_range": analysis.fit.fit_range,
"energy_power": 2,
"flux_unit": "erg-1 cm-2 s-1",
}
axes = analysis.spectrum_result.plot(**opts)
CrabSpectrum().model.plot(ax=axes[0], **opts)
# ## Exercises
#
# Rerun the analysis, changing some aspects of the analysis as you like:
#
# * only use one or two observations
# * a different spectral model
# * different config options for the spectral analysis
# * different energy binning for the spectral point computation
#
# Observe how the measured spectrum changes.
def plot_fit_results(tool):
"""plot the SEDs result of the gammapy / sherpa fit
for comparison with the literature we choose only the Mayer spectrum
Here we plot the butterfly as a result of the multivariate sampling
with the 68% containment in flux
"""
fig, ax = plt.subplots()
model_meyer_ref = CrabSpectrum("meyer").model
model_meyer_ref.plot(
[10 * u.GeV, 100 * u.TeV],
energy_power=2,
flux_unit="erg-1 cm-2 s-1",
ls=":",
lw=2.2,
color="#555555",
label="Meyer et al. (2010)",
)
# where to take the results, configurations for the individual butterflies
instruments = ["fermi", "magic", "veritas", "fact", "hess", "joint"]
labels = ["Fermi-LAT", "MAGIC", "VERITAS", "FACT", "H.E.S.S.", "joint fit"]
lss = ["--", "--", "--", "--", "--", "-"]
colors = COLORS
# with one loop we realize all the butterfly plots
for instrument, label, color, ls in zip(instruments, labels, colors, lss):
path = (
config.repo_path /
f"results/fit/{tool}/{instrument}/fit_results_logparabola.yaml")
if not path.exists():
log.warning(f"Missing: {path} . Skipping.")
continue
results = load_yaml(path)
parameters = results["parameters"]
model_lp = LogParabola.from_log10(
amplitude=parameters[0]["value"] * u.Unit(parameters[0]["unit"]),
reference=parameters[1]["value"] * u.Unit(parameters[1]["unit"]),
alpha=parameters[2]["value"] * u.Unit(parameters[2]["unit"]),
beta=parameters[3]["value"] * u.Unit(parameters[3]["unit"]),
)
# energy range for the plot
dataset = config.get_dataset(instrument)
energy_range = dataset.energy_range
# just in case of the joint fit put a thicker line and a less transparent butterfly
if instrument == "joint":
model_lp.plot(
energy_range,
energy_power=2,
flux_unit="erg-1 cm-2 s-1",
ls=ls,
lw=3,
color=color,
label=label,
)
else:
model_lp.plot(
energy_range,
energy_power=2,
flux_unit="erg-1 cm-2 s-1",
ls=ls,
lw=2.2,
color=color,
label=label,
)
# read the butterfly from the multivariate sampling results
table_path = Path(
f"{config.repo_path}/results/figures/stat_err/{instrument}_flux_errorband.dat"
)
log.info(f"reading butterfly values from {table_path}")
t = Table.read(table_path, format="ascii.ecsv")
energies = t["energies"].data * t["energies"].unit
flux_lo = t["flux_lo"].data * t["flux_lo"].unit
flux_hi = t["flux_hi"].data * t["flux_hi"].unit
if instrument == "joint":
alpha = 0.38
else:
alpha = 0.28
plt.fill_between(
energies.to("TeV"),
(energies**2 * flux_lo).to("erg cm-2 s-1"),
(energies**2 * flux_hi).to("erg cm-2 s-1"),
color=color,
alpha=alpha,
label="",
)
ax.legend(fontsize=FONTSIZE)
ax.set_ylim([1e-12, 2e-10])
ax.set_xlabel(E_UNIT_LABEL, size=FONTSIZE)
ax.set_ylabel(SED_UNIT_LABEL, size=FONTSIZE)
# make axis thicker
for axis in ["top", "bottom", "left", "right"]:
ax.spines[axis].set_linewidth(1.6)
ax.tick_params("both",
length=7,
width=1.6,
which="major",
labelsize=FONTSIZE)
ax.tick_params("both",
length=4,
width=1.6,
which="minor",
labelsize=FONTSIZE)
plt.tight_layout()
filename = f"results/figures/crab_sed_{tool}_fit.png"
filename_pdf = f"results/figures/crab_sed_{tool}_fit.pdf"
log.info(f"Writing {filename}")
fig.savefig(filename)
fig.savefig(filename_pdf)
print(lc.table.colnames)
# In[ ]:
lc.table["time_min", "time_max", "flux", "flux_err"]
# In[ ]:
lc.plot()
# In[ ]:
# Let's compare to the expected flux of this source
from gammapy.spectrum import CrabSpectrum
crab_spec = CrabSpectrum().model
crab_flux = crab_spec.integral(*energy_range).to("cm-2 s-1")
crab_flux
# In[ ]:
ax = lc.plot(marker="o", lw=2)
ax.hlines(
crab_flux.value,
xmin=lc.table["time_min"].min(),
xmax=lc.table["time_max"].max(),
)
# ## Exercises
#
# * Change the assumed spectral model shape (e.g. to a steeper power-law), and see how the integral flux estimate for the lightcurve changes.
def main():
# =========================================================================
# READ INPUT FROM CLI AND CONFIGURATION FILE
# =========================================================================
# INPUT FROM CLI
parser = argparse.ArgumentParser(description="Produce DL3 data from DL2.")
parser.add_argument(
"--config_file",
type=str,
required=True,
help="A configuration file like pyirf/resources/performance.yaml .",
)
parser.add_argument(
"--obs_time",
type=str,
required=True,
help=
"An observation time given as a string in astropy format e.g. '50h' or '30min'",
)
parser.add_argument(
"--pipeline",
type=str,
required=True,
help="Name of the pipeline that has produced the DL2 files.",
)
parser.add_argument("--debug",
action="store_true",
help="Print debugging information.")
args = parser.parse_args()
# INPUT FROM THE CONFIGURATION FILE
cfg = load_config(args.config_file)
# Add obs. time to the configuration file
obs_time = u.Quantity(args.obs_time)
cfg["analysis"]["obs_time"] = {
"value": obs_time.value,
"unit": obs_time.unit.to_string("fits"),
}
# Get input directory
indir = cfg["general"]["indir"]
# Get template of the input file(s)
template_input_file = cfg["general"]["template_input_file"]
# Get output directory
outdir = os.path.join(
cfg["general"]["outdir"],
"irf_{}_Time{}{}".format(
args.pipeline,
cfg["analysis"]["obs_time"]["value"],
cfg["analysis"]["obs_time"]["unit"],
),
) # and create it if necessary
os.makedirs(outdir, exist_ok=True)
# =========================================================================
# READ DL2 DATA AND STORE IT ACCORDING TO GADF
# =========================================================================
# Load FITS data
particles = ["gamma", "electron", "proton"]
evt_dict = dict() # Contain DL2 file for each type of particle
for particle in particles:
if args.debug:
print(f"Loading {particle} DL2 data...")
infile = os.path.join(indir, template_input_file.format(particle))
evt_dict[particle] = read_FITS(config=cfg,
infile=infile,
pipeline=args.pipeline,
debug=args.debug)
# =========================================================================
# PRELIMINARY OPERATIONS FOR SPECIFIC PIPELINES
# =========================================================================
# Some pipelines could provide some of the DL2 data in different ways
# After this part, DL2 data is supposed to be equivalent, regardless
# of the original pipeline.
# Later we should move this out of here, perhaps under a "utils" module.
if args.pipeline == "EventDisplay":
# EventDisplay provides true and reconstructed directions, so we
# calculate THETA here and we add it to the tables.
for particle in particles:
THETA = angular_separation(
evt_dict[particle]["TRUE_AZ"],
evt_dict[particle]["TRUE_ALT"],
evt_dict[particle]["AZ"],
evt_dict[particle]["ALT"],
) # in degrees
# Add THETA column
evt_dict[particle]["THETA"] = THETA
# =========================================================================
# REST OF THE OPERATIONS (TO BE REFACTORED)
# =========================================================================
# Apply offset cut to proton and electron
for particle in ["electron", "proton"]:
# There seems to be a problem in using pandas from FITS data
# ValueError: Big-endian buffer not supported on little-endian compiler
# I convert to astropy table....
# should we use only those?
evt_dict[particle] = Table.from_pandas(evt_dict[particle])
if args.debug:
print(particle)
# print(evt_dict[particle].head(n=5))
print(evt_dict[particle])
# print('Initial stat: {} {}'.format(len(evt_dict[particle]), particle))
mask_theta = (evt_dict[particle]["THETA"] <
cfg["particle_information"][particle]["offset_cut"])
evt_dict[particle] = evt_dict[particle][mask_theta]
# PANDAS EQUIVALENT
# evt_dict[particle] = evt_dict[particle].query(
# "THETA <= {}".format(cfg["particle_information"][particle]["offset_cut"])
# )
# Add required data in configuration file for future computation
for particle in particles:
n_files = cfg["particle_information"][particle]["n_files"]
print(f"{n_files} files for {particle}")
cfg["particle_information"][particle]["n_files"] = len(
np.unique(evt_dict[particle]["OBS_ID"]))
cfg["particle_information"][particle]["n_simulated"] = (
cfg["particle_information"][particle]["n_files"] *
cfg["particle_information"][particle]["n_events_per_file"])
# Define model for the particles
model_dict = {
"gamma": CrabSpectrum("hegra").model,
"proton": cosmic_ray_flux,
"electron": cosmic_ray_flux,
}
# Reco energy binning
cfg_binning = cfg["analysis"]["ereco_binning"]
ereco = (np.logspace(
np.log10(cfg_binning["emin"]),
np.log10(cfg_binning["emax"]),
cfg_binning["nbin"] + 1,
) * u.TeV)
# Handle theta square cut optimisation
# (compute 68 % containment radius PSF if necessary)
thsq_opt_type = cfg["analysis"]["thsq_opt"]["type"]
if thsq_opt_type == "fixed":
thsq_values = np.array([cfg["analysis"]["thsq_opt"]["value"]]) * u.deg
print("Using fixed theta cut: {}".format(thsq_values))
elif thsq_opt_type == "opti":
thsq_values = np.arange(0.05, 0.40, 0.01) * u.deg
print("Optimising theta cut for: {}".format(thsq_values))
elif thsq_opt_type == "r68":
print("Using R68% theta cut")
print("Computing...")
cfg_binning = cfg["analysis"]["ereco_binning"]
ereco = (np.logspace(
np.log10(cfg_binning["emin"]),
np.log10(cfg_binning["emax"]),
cfg_binning["nbin"] + 1,
) * u.TeV)
radius = 68
thsq_values = list()
# There seems to be a problem in using pandas from FITS data
# ValueError: Big-endian buffer not supported on little-endian compiler
# I convert to astropy table....
# should we use only those?
evt_dict["gamma"] = Table.from_pandas(evt_dict["gamma"])
if args.debug:
print("GAMMAS")
# print(evt_dict["gamma"].head(n=5))
print(evt_dict["gamma"])
for ibin in range(len(ereco) - 1):
emin = ereco[ibin]
emax = ereco[ibin + 1]
# PANDAS EQUIVALENT
# energy_query = "reco_energy > {} and reco_energy <= {}".format(
# emin.value, emax.value
# )
# data = evt_dict["gamma"].query(energy_query).copy()
mask_energy = (evt_dict["gamma"]["ENERGY"] > emin.value) & (
evt_dict["gamma"]["ENERGY"] < emax.value)
data = evt_dict["gamma"][mask_energy]
min_stat = 0
if len(data) <= min_stat:
print(" ==> Not enough statistics:")
print("To be handled...")
thsq_values.append(0.3)
continue
# import sys
# sys.exit()
psf = np.percentile(data["THETA"], radius)
# psf_err = psf / np.sqrt(len(data)) # not used after?
thsq_values.append(psf)
thsq_values = np.array(thsq_values) * u.deg
# Set 0.05 as a lower value
idx = np.where(thsq_values.value < 0.05)
thsq_values[idx] = 0.05 * u.deg
print("Using theta cut: {}".format(thsq_values))
# Cuts optimisation
print("### Finding best cuts...")
cut_optimiser = CutsOptimisation(config=cfg,
evt_dict=evt_dict,
verbose_level=0)
# Weight events
print("- Weighting events...")
cut_optimiser.weight_events(
model_dict=model_dict,
# colname_mc_energy=cfg["column_definition"]["TRUE_ENERGY"],
colname_mc_energy="TRUE_ENERGY",
)
# Find best cutoff to reach best sensitivity
print("- Estimating cutoffs...")
cut_optimiser.find_best_cutoff(energy_values=ereco,
angular_values=thsq_values)
# Save results and auxiliary data for diagnostic
print("- Saving results to disk...")
cut_optimiser.write_results(outdir,
"{}.fits".format(
cfg["general"]["output_table_name"]),
format="fits")
# Cuts diagnostic
print("### Building cut diagnostics...")
cut_diagnostic = CutsDiagnostic(config=cfg, indir=outdir)
cut_diagnostic.plot_optimisation_summary()
cut_diagnostic.plot_diagnostics()
# Apply cuts and save data
print("### Applying cuts to data...")
cut_applicator = CutsApplicator(config=cfg,
evt_dict=evt_dict,
outdir=outdir)
cut_applicator.apply_cuts(args.debug)
# Irf Maker
print("### Building IRF...")
irf_maker = IrfMaker(config=cfg, evt_dict=evt_dict, outdir=outdir)
irf_maker.build_irf(thsq_values)
# Sensitivity maker
print("### Estimating sensitivity...")
sensitivity_maker = SensitivityMaker(config=cfg, outdir=outdir)
sensitivity_maker.load_irf()
sensitivity_maker.estimate_sensitivity()
from gammapy.datasets import load_crab_flux_points
from gammapy.spectrum import CrabSpectrum
# Plot flux points
for component in ['pulsar', 'nebula']:
table = load_crab_flux_points(component=component)
x = table['energy'].data
y = table['energy_flux'].data
yerr_lo = table['energy_flux_err_lo'].data
yerr_hi = table['energy_flux_err_hi'].data
plt.errorbar(x, y, yerr=(yerr_lo, yerr_hi), fmt='o', label=component)
# Plot SED model
energy = np.logspace(2, 8, 100) * u.MeV
crab = CrabSpectrum(reference='meyer')
flux = crab.model(energy)
energy_flux = (energy ** 2 * flux).to('erg cm^-2 s^-1')
plt.plot(energy.value, energy_flux.value, label='Meyer (2010) model', lw=3)
plt.title('Crab pulsar and nebula spectral energy distribution (SED)')
plt.xlim((3e-14, 3e8))
plt.ylim((3e-13, 3e-7))
plt.xlabel('Energy (MeV)')
plt.ylabel('E^2 dN/dE (erg cm^-2 s^-1)')
plt.legend(loc='upper center', ncol=3)
plt.grid()
plt.loglog()
plt.savefig('crab_mwl.png')
import numpy as np
import click
import matplotlib.pyplot as plt
import os
from gammapy.spectrum import CrabSpectrum
import astropy.units as u
def model(amplitude, alpha, beta):
amplitude *= 1E-11
return lambda xx: amplitude * np.power(xx, (-alpha - beta * np.log10(xx)))
magic_model = CrabSpectrum('magic_lp').model
meyer_model = CrabSpectrum('meyer').model
fact_model = model(3.49, 2.54, 0.42)
magic_joint_model = model(4.15, 2.6, 0.44)
hess_joint_model = model(4.47, 2.39, 0.37)
@click.command()
@click.argument('input_dir', type=click.Path(dir_okay=True, file_okay=False))
@click.option('--output',
'-o',
type=click.Path(dir_okay=False, file_okay=True))
@click.option('--color', default='crimson')
def main(input_dir, output, color):
alphas = []
betas = []
def main():
# Read arguments
parser = argparse.ArgumentParser(description="Make performance files")
parser.add_argument("--config_file", type=str, required=True, help="")
parser.add_argument(
"--obs_time",
type=str,
required=True,
help="Observation time, should be given as a string, value and astropy unit separated by an empty space",
)
mode_group = parser.add_mutually_exclusive_group()
mode_group.add_argument(
"--wave",
dest="mode",
action="store_const",
const="wave",
default="tail",
help="if set, use wavelet cleaning",
)
mode_group.add_argument(
"--tail",
dest="mode",
action="store_const",
const="tail",
help="if set, use tail cleaning, otherwise wavelets",
)
args = parser.parse_args()
# Read configuration file
cfg = load_config(args.config_file)
# Add obs. time in configuration file
str_obs_time = args.obs_time.split()
cfg["analysis"]["obs_time"] = {
"value": float(str_obs_time[0]),
"unit": str(str_obs_time[-1]),
}
# Create output directory if necessary
outdir = os.path.join(
cfg["general"]["outdir"],
"irf_{}_ThSq_{}_Time{:.2f}{}".format(
args.mode,
cfg["analysis"]["thsq_opt"]["type"],
cfg["analysis"]["obs_time"]["value"],
cfg["analysis"]["obs_time"]["unit"],
),
)
if not os.path.exists(outdir):
os.makedirs(outdir)
indir = cfg["general"]["indir"]
template_input_file = cfg["general"]["template_input_file"]
# Load data
particles = ["gamma", "electron", "proton"]
evt_dict = dict() # Contain DL2 file for each type of particle
for particle in particles:
# template looks like dl2_{}_{}_merged.h5
infile = os.path.join(indir, template_input_file.format(args.mode, particle))
evt_dict[particle] = pd.read_hdf(infile, key="reco_events")
# Apply offset cut to proton and electron
for particle in ["electron", "proton"]:
# print('Initial stat: {} {}'.format(len(evt_dict[particle]), particle))
evt_dict[particle] = evt_dict[particle].query('offset <= {}'.format(
cfg['analysis']['max_bg_radius'])
)
# Add required data in configuration file for future computation
for particle in particles:
cfg['particle_information'][particle]['n_files'] = \
len(np.unique(evt_dict[particle]['obs_id']))
cfg['particle_information'][particle]['n_simulated'] = \
cfg['particle_information'][particle]['n_files'] * cfg['particle_information'][particle]['num_showers'] * cfg['particle_information'][particle]['num_use']
# Define model for the particles
model_dict = {
"gamma": CrabSpectrum("hegra").model,
"proton": cosmic_ray_flux,
"electron": cosmic_ray_flux,
}
# Reco energy binning
cfg_binning = cfg["analysis"]["ereco_binning"]
ereco = (
np.logspace(
np.log10(cfg_binning["emin"]),
np.log10(cfg_binning["emax"]),
cfg_binning["nbin"] + 1,
)
* u.TeV
)
# Handle theta square cut optimisation
# (compute 68 % containment radius PSF if necessary)
thsq_opt_type = cfg["analysis"]["thsq_opt"]["type"]
if thsq_opt_type in "fixed":
thsq_values = np.array([cfg["analysis"]["thsq_opt"]["value"]]) * u.deg
print("Using fixed theta cut: {}".format(thsq_values))
elif thsq_opt_type in "opti":
thsq_values = np.arange(0.05, 0.40, 0.01) * u.deg
print("Optimising theta cut for: {}".format(thsq_values))
elif thsq_opt_type in "r68":
print("Using R68% theta cut")
print("Computing...")
cfg_binning = cfg["analysis"]["ereco_binning"]
ereco = (
np.logspace(
np.log10(cfg_binning["emin"]),
np.log10(cfg_binning["emax"]),
cfg_binning["nbin"] + 1,
)
* u.TeV
)
radius = 68
thsq_values = list()
for ibin in range(len(ereco) - 1):
emin = ereco[ibin]
emax = ereco[ibin + 1]
energy_query = "reco_energy > {} and reco_energy <= {}".format(
emin.value, emax.value
)
data = evt_dict["gamma"].query(energy_query).copy()
min_stat = 0
if len(data) <= min_stat:
print(" ==> Not enough statistics:")
print("To be handled...")
thsq_values.append(0.3)
continue
# import sys
# sys.exit()
psf = np.percentile(data["offset"], radius)
psf_err = psf / np.sqrt(len(data))
thsq_values.append(psf)
thsq_values = np.array(thsq_values) * u.deg
# Set 0.05 as a lower value
idx = np.where(thsq_values.value < 0.05)
thsq_values[idx] = 0.05 * u.deg
print("Using theta cut: {}".format(thsq_values))
# Cuts optimisation
print("### Finding best cuts...")
cut_optimiser = CutsOptimisation(config=cfg, evt_dict=evt_dict, verbose_level=0)
# Weight events
print("- Weighting events...")
cut_optimiser.weight_events(
model_dict=model_dict, colname_mc_energy=cfg["column_definition"]["mc_energy"]
)
# Find best cutoff to reach best sensitivity
print("- Estimating cutoffs...")
cut_optimiser.find_best_cutoff(energy_values=ereco, angular_values=thsq_values)
# Save results and auxiliary data for diagnostic
print("- Saving results to disk...")
cut_optimiser.write_results(
outdir, "{}.fits".format(cfg["general"]["output_table_name"]), format="fits"
)
# Cuts diagnostic
print("### Building cut diagnostics...")
cut_diagnostic = CutsDiagnostic(config=cfg, indir=outdir)
cut_diagnostic.plot_optimisation_summary()
cut_diagnostic.plot_diagnostics()
# Apply cuts and save data
print("### Applying cuts to data...")
cut_applicator = CutsApplicator(config=cfg, evt_dict=evt_dict, outdir=outdir)
cut_applicator.apply_cuts()
# Irf Maker
print("### Building IRF...")
irf_maker = IrfMaker(config=cfg, evt_dict=evt_dict, outdir=outdir)
irf_maker.build_irf()
# Sensitivity maker
print("### Estimating sensitivity...")
sensitivity_maker = SensitivityMaker(config=cfg, outdir=outdir)
sensitivity_maker.load_irf()
sensitivity_maker.estimate_sensitivity()
def plot_spectra(sampler,
mle_result,
fit_range=[0.03, 30] * u.TeV,
min_sample=50):
joint_model = Log10Parabola(
amplitude=3.78 * 1e-11 * u.Unit('cm-2 s-1 TeV-1'),
reference=1 * u.Unit('TeV'),
alpha=2.49 * u.Unit(''),
beta=0.22 * u.Unit(''),
)
joint_model.plot(energy_range=fit_range,
energy_power=2,
color='black',
label='joint')
r = np.median(sampler.chain[:, min_sample:, :3], axis=(0, 1))
fitted_model = Log10Parabola(
amplitude=r[0] * 1e-11 * u.Unit('cm-2 s-1 TeV-1'),
reference=1 * u.Unit('TeV'),
alpha=r[1] * u.Unit(''),
beta=r[2] * u.Unit(''),
)
fitted_model.plot(energy_range=fit_range,
energy_power=2,
color='crimson',
label='mcmc')
mle_model = Log10Parabola(
amplitude=mle_result.x[0] * 1e-11 * u.Unit('cm-2 s-1 TeV-1'),
reference=1 * u.Unit('TeV'),
alpha=mle_result.x[1] * u.Unit(''),
beta=mle_result.x[2] * u.Unit(''),
)
mle_model.plot(energy_range=fit_range,
energy_power=2,
color='orange',
ls='--',
label='mle')
fact_model = Log10Parabola(
amplitude=3.47 * 1e-11 * u.Unit('cm-2 s-1 TeV-1'),
reference=1 * u.Unit('TeV'),
alpha=2.56 * u.Unit(''),
beta=0.4 * u.Unit(''),
)
fact_model.plot(energy_range=fit_range,
energy_power=2,
color='gray',
label='fact')
magic_model = Log10Parabola(
amplitude=4.20 * 1e-11 * u.Unit('cm-2 s-1 TeV-1'),
reference=1 * u.Unit('TeV'),
alpha=2.58 * u.Unit(''),
beta=0.43 * u.Unit(''),
)
magic_model.plot(energy_range=fit_range,
energy_power=2,
color='gray',
ls='--',
label='magic')
CrabSpectrum(reference='meyer').model.plot(energy_range=[0.01, 100] *
u.TeV,
energy_power=2,
color='black',
ls=':')