def test_compute_flux_points_dnde_exp(method):
"""
Tests against analytical result or result from gammapy.spectrum.powerlaw.
"""
model = ExpTestModel()
e_min = [1.0, 10.0] * u.TeV
e_max = [10.0, 100.0] * u.TeV
table = Table()
table.meta["SED_TYPE"] = "flux"
table["e_min"] = e_min
table["e_max"] = e_max
flux = model.integral(e_min, e_max)
table["flux"] = flux
if method == "log_center":
e_ref = np.sqrt(e_min * e_max)
elif method == "table":
e_ref = [2.0, 20.0] * u.TeV
table["e_ref"] = e_ref
elif method == "lafferty":
e_ref = FluxPoints._e_ref_lafferty(model, e_min, e_max)
result = FluxPoints(table).to_sed_type("dnde", model=model, method=method)
# Test energy
actual = result.e_ref
assert_quantity_allclose(actual, e_ref, rtol=1e-8)
# Test flux
actual = result.table["dnde"].quantity
desired = model(e_ref)
assert_quantity_allclose(actual, desired, rtol=1e-8)
def test_dnde_from_flux():
"""Tests y-value normalization adjustment method.
"""
e_min = np.array([10, 20, 30, 40])
e_max = np.array([20, 30, 40, 50])
flux = np.array([42, 52, 62, 72]) # 'True' integral flux in this test bin
# Get values
model = XSqrTestModel()
e_ref = FluxPoints._e_ref_lafferty(model, e_min, e_max)
dnde = FluxPoints._dnde_from_flux(flux,
model,
e_ref,
e_min,
e_max,
pwl_approx=False)
# Set up test case comparison
dnde_model = model(e_ref)
# Test comparison result
desired = model.integral(e_min, e_max)
# Test output result
actual = flux * (dnde_model / dnde)
# Compare
assert_allclose(actual, desired, rtol=1e-6)
def make_plots(reference, result):
fpoints_ref = FluxPoints.read(reference)
fpoints_res = FluxPoints.read(result)
fig = plt.figure(figsize=(7, 5))
opts = {"energy_power": 2}
fpoints_ref.plot(**opts, label="reference")
fpoints_res.plot(**opts)
plt.legend()
return fig
def flux_points(self):
"""Flux points (`~gammapy.spectrum.FluxPoints`)."""
table = Table()
table.meta["SED_TYPE"] = "flux"
table["e_min"] = self._ebounds[:-1]
table["e_max"] = self._ebounds[1:]
flux = self.data["Flux_Band"]
flux_err = self.data["Unc_Flux_Band"]
e2dnde = self.data["nuFnu"]
table["flux"] = flux
table["flux_errn"] = np.abs(flux_err[:, 0])
table["flux_errp"] = flux_err[:, 1]
table["e2dnde"] = e2dnde
table["e2dnde_errn"] = np.abs(e2dnde * flux_err[:, 0] / flux)
table["e2dnde_errp"] = e2dnde * flux_err[:, 1] / flux
is_ul = np.isnan(table["flux_errn"])
table["is_ul"] = is_ul
# handle upper limits
table["flux_ul"] = np.nan * flux_err.unit
flux_ul = compute_flux_points_ul(table["flux"], table["flux_errp"])
table["flux_ul"][is_ul] = flux_ul[is_ul]
table["e2dnde_ul"] = np.nan * e2dnde.unit
e2dnde_ul = compute_flux_points_ul(table["e2dnde"],
table["e2dnde_errp"])
table["e2dnde_ul"][is_ul] = e2dnde_ul[is_ul]
# Square root of test statistic
table["sqrt_ts"] = self.data["Sqrt_TS_Band"]
return FluxPoints(table)
def flux_points_e2dnde(model):
e_ref = [np.sqrt(10), np.sqrt(10 * 100)] * u.TeV
table = Table()
table.meta["SED_TYPE"] = "e2dnde"
table["e_ref"] = e_ref
table["e2dnde"] = (model(e_ref) * e_ref**2).to("erg cm-2 s-1")
return FluxPoints(table)
def flux_points(self):
"""Differential flux points (`~gammapy.spectrum.FluxPoints`)."""
d = self.data
table = Table()
table.meta["SED_TYPE"] = "dnde"
self._add_source_meta(table)
valid = np.isfinite(d["sed_e_ref"].value)
if valid.sum() == 0:
return None
table["e_ref"] = d["sed_e_ref"]
table["e_min"] = d["sed_e_min"]
table["e_max"] = d["sed_e_max"]
table["dnde"] = d["sed_dnde"]
table["dnde_err"] = d["sed_dnde_err"]
table["dnde_errn"] = d["sed_dnde_errn"]
table["dnde_errp"] = d["sed_dnde_errp"]
table["dnde_ul"] = d["sed_dnde_ul"]
# Only keep rows that actually contain information
table = table[valid]
# Only keep columns that actually contain information
def _del_nan_col(table, colname):
if np.isfinite(table[colname]).sum() == 0:
del table[colname]
for colname in table.colnames:
_del_nan_col(table, colname)
return FluxPoints(table)
def flux_points_dnde(model):
e_ref = [np.sqrt(10), np.sqrt(10 * 100)] * u.TeV
table = Table()
table.meta["SED_TYPE"] = "dnde"
table["e_ref"] = e_ref
table["dnde"] = model(e_ref)
return FluxPoints(table)
def dataset():
path = "$GAMMAPY_DATA/tests/spectrum/flux_points/diff_flux_points.fits"
data = FluxPoints.read(path)
data.table["e_ref"] = data.e_ref.to("TeV")
model = SkyModel(spectral_model=PowerLawSpectralModel(
index=2.3, amplitude="2e-13 cm-2 s-1 TeV-1", reference="1 TeV"))
dataset = FluxPointsDataset(model, data)
return dataset
def flux_points_flux(model):
e_min = [1, 10] * u.TeV
e_max = [10, 100] * u.TeV
table = Table()
table.meta["SED_TYPE"] = "flux"
table["e_min"] = e_min
table["e_max"] = e_max
table["flux"] = model.integral(e_min, e_max)
return FluxPoints(table)
def read_regions(self):
for kr in self.ROIs_sel:
filedata = Path(self.resdir + "/3FHL_ROI_num" + str(kr) +
"_datasets.yaml")
filemodel = Path(self.resdir + "/3FHL_ROI_num" + str(kr) +
"_models.yaml")
try:
dataset = list(Datasets.from_yaml(filedata, filemodel))[0]
except (FileNotFoundError, IOError):
continue
pars = dataset.parameters
pars.covariance = np.load(self.resdir + "/" + dataset.name +
"_covariance.npy")
infos = np.load(self.resdir + "/3FHL_ROI_num" + str(kr) +
"_fit_infos.npz")
self.diags["message"].append(infos["message"])
self.diags["stat"].append(infos["stat"])
if self.savefig:
self.plot_maps(dataset)
for model in list(dataset.model):
if (self.FHL3[model.name].data["ROI_num"] == kr
and self.FHL3[model.name].data["Signif_Avg"] >=
self.sig_cut):
model.spatial_model.parameters.covariance = pars.get_subcovariance(
model.spatial_model.parameters)
model.spectral_model.parameters.covariance = pars.get_subcovariance(
model.spectral_model.parameters)
dataset.background_model.parameters.covariance = pars.get_subcovariance(
dataset.background_model.parameters)
res_spec = model.spectral_model
cat_spec = self.FHL3[model.name].spectral_model()
res_fp = FluxPoints.read(self.resdir + "/" + model.name +
"_flux_points.fits")
res_fp.table["is_ul"] = res_fp.table["ts"] < 1.0
cat_fp = self.FHL3[model.name].flux_points.to_sed_type(
"dnde")
self.update_spec_diags(dataset, model, cat_spec, res_spec,
cat_fp, res_fp)
if self.savefig:
self.plot_spec(kr, model, cat_spec, res_spec, cat_fp,
res_fp)
def test_e_ref_lafferty():
"""
Tests Lafferty & Wyatt x-point method.
Using input function g(x) = 10^4 exp(-6x) against
check values from paper Lafferty & Wyatt. Nucl. Instr. and Meth. in Phys.
Res. A 355 (1995) 541-547, p. 542 Table 1
"""
# These are the results from the paper
desired = np.array([0.048, 0.190, 0.428, 0.762])
model = LWTestModel()
e_min = np.array([0.0, 0.1, 0.3, 0.6])
e_max = np.array([0.1, 0.3, 0.6, 1.0])
actual = FluxPoints._e_ref_lafferty(model, e_min, e_max)
assert_allclose(actual, desired, atol=1e-3)
def run(self, e_ref, e_min, e_max, steps="all"):
"""Run light curve extraction.
Normalize integral and energy flux between emin and emax.
Parameters
----------
e_ref : `~astropy.unit.Quantity`
reference energy of dnde flux normalization
e_min : `~astropy.unit.Quantity`
minimum energy of integral and energy flux normalization interval
e_max : `~astropy.unit.Quantity`
minimum energy of integral and energy flux normalization interval
steps : list of str
Which steps to execute. Available options are:
* "err": estimate symmetric error.
* "errn-errp": estimate asymmetric errors.
* "ul": estimate upper limits.
* "ts": estimate ts and sqrt(ts) values.
* "norm-scan": estimate likelihood profiles.
By default all steps are executed.
Returns
-------
lightcurve : `~gammapy.time.LightCurve`
the Light Curve object
"""
self.e_ref = e_ref
self.e_min = e_min
self.e_max = e_max
rows = []
for dataset in self.datasets.datasets:
row = {
"time_min": dataset.counts.meta["t_start"].mjd,
"time_max": dataset.counts.meta["t_stop"].mjd,
}
row.update(self.estimate_time_bin_flux(dataset, steps))
rows.append(row)
meta = OrderedDict([("SED_TYPE", "likelihood")])
table = table_from_row_data(rows=rows, meta=meta)
table = FluxPoints(table).to_sed_type("flux").table
return LightCurve(table)
def flux_points(self):
"""Flux points (`~gammapy.spectrum.FluxPoints`)."""
table = Table()
table.meta["SED_TYPE"] = "dnde"
mask = ~np.isnan(self.data["Flux_Points_Energy"])
table["e_ref"] = self.data["Flux_Points_Energy"][mask]
table["e_min"] = self.data["Flux_Points_Energy_Min"][mask]
table["e_max"] = self.data["Flux_Points_Energy_Max"][mask]
table["dnde"] = self.data["Flux_Points_Flux"][mask]
table["dnde_errn"] = self.data["Flux_Points_Flux_Err_Lo"][mask]
table["dnde_errp"] = self.data["Flux_Points_Flux_Err_Hi"][mask]
table["dnde_ul"] = self.data["Flux_Points_Flux_UL"][mask]
table["is_ul"] = self.data["Flux_Points_Flux_Is_UL"][mask].astype(
"bool")
return FluxPoints(table)
def flux_points(self):
"""Integral flux points (`~gammapy.spectrum.FluxPoints`)."""
table = Table()
table.meta["SED_TYPE"] = "flux"
table["e_min"] = self._ebounds[:-1]
table["e_max"] = self._ebounds[1:]
table["flux"] = self._get_flux_values("Flux")
flux_err = self._get_flux_values("Unc_Flux")
table["flux_errn"] = np.abs(flux_err[:, 0])
table["flux_errp"] = flux_err[:, 1]
# handle upper limits
is_ul = np.isnan(table["flux_errn"])
table["is_ul"] = is_ul
table["flux_ul"] = np.nan * flux_err.unit
flux_ul = compute_flux_points_ul(table["flux"], table["flux_errp"])
table["flux_ul"][is_ul] = flux_ul[is_ul]
return FluxPoints(table)
def test_flux_point_dataset_serialization(tmp_path):
path = "$GAMMAPY_DATA/tests/spectrum/flux_points/diff_flux_points.fits"
data = FluxPoints.read(path)
data.table["e_ref"] = data.e_ref.to("TeV")
spectral_model = PowerLawSpectralModel(index=2.3,
amplitude="2e-13 cm-2 s-1 TeV-1",
reference="1 TeV")
model = SkyModel(spectral_model=spectral_model, name="test_model")
dataset = FluxPointsDataset(model, data, name="test_dataset")
Datasets([dataset]).write(tmp_path, prefix="tmp")
datasets = Datasets.read(tmp_path / "tmp_datasets.yaml",
tmp_path / "tmp_models.yaml")
new_dataset = datasets[0]
assert_allclose(new_dataset.data.table["dnde"], dataset.data.table["dnde"],
1e-4)
if dataset.mask_fit is None:
assert np.all(new_dataset.mask_fit == dataset.mask_safe)
assert np.all(new_dataset.mask_safe == dataset.mask_safe)
assert new_dataset.name == "test_dataset"
def test_flux_point_dataset_serialization(tmp_path):
path = "$GAMMAPY_DATA/tests/spectrum/flux_points/diff_flux_points.fits"
data = FluxPoints.read(path)
data.table["e_ref"] = data.e_ref.to("TeV")
# TODO: remove duplicate definition this once model is redefine as skymodel
spatial_model = ConstantSpatialModel()
spectral_model = PowerLawSpectralModel(index=2.3,
amplitude="2e-13 cm-2 s-1 TeV-1",
reference="1 TeV")
model = SkyModel(spatial_model, spectral_model, name="test_model")
dataset = FluxPointsDataset(SkyModels([model]), data, name="test_dataset")
Datasets([dataset]).to_yaml(tmp_path, prefix="tmp")
datasets = Datasets.from_yaml(tmp_path / "tmp_datasets.yaml",
tmp_path / "tmp_models.yaml")
new_dataset = datasets[0]
assert_allclose(new_dataset.data.table["dnde"], dataset.data.table["dnde"],
1e-4)
if dataset.mask_fit is None:
assert np.all(new_dataset.mask_fit == dataset.mask_safe)
assert np.all(new_dataset.mask_safe == dataset.mask_safe)
assert new_dataset.name == "test_dataset"
def flux_points(self):
"""Flux points (`~gammapy.spectrum.FluxPoints`)."""
table = Table()
table.meta["SED_TYPE"] = "flux"
table["e_min"] = self._ebounds[:-1]
table["e_max"] = self._ebounds[1:]
flux = self._get_flux_values("Flux")
flux_err = self._get_flux_values("Unc_Flux")
table["flux"] = flux
table["flux_errn"] = np.abs(flux_err[:, 0])
table["flux_errp"] = flux_err[:, 1]
nuFnu = self._get_flux_values("nuFnu", "erg cm-2 s-1")
table["e2dnde"] = nuFnu
table["e2dnde_errn"] = np.abs(nuFnu * flux_err[:, 0] / flux)
table["e2dnde_errp"] = nuFnu * flux_err[:, 1] / flux
is_ul = np.isnan(table["flux_errn"])
table["is_ul"] = is_ul
# handle upper limits
table["flux_ul"] = np.nan * flux_err.unit
flux_ul = compute_flux_points_ul(table["flux"], table["flux_errp"])
table["flux_ul"][is_ul] = flux_ul[is_ul]
# handle upper limits
table["e2dnde_ul"] = np.nan * nuFnu.unit
e2dnde_ul = compute_flux_points_ul(table["e2dnde"],
table["e2dnde_errp"])
table["e2dnde_ul"][is_ul] = e2dnde_ul[is_ul]
# Square root of test statistic
table["sqrt_TS"] = [
self.data["Sqrt_TS" + _] for _ in self._ebounds_suffix
]
return FluxPoints(table)
def test_write_ecsv(self, tmpdir, flux_points):
filename = tmpdir / "flux_points.ecsv"
flux_points.write(filename)
actual = FluxPoints.read(filename)
assert str(flux_points) == str(actual)
def test_stack(self, flux_points):
stacked = FluxPoints.stack([flux_points, flux_points])
assert len(stacked.table) == 2 * len(flux_points.table)
assert stacked.sed_type == flux_points.sed_type
def test_write_ecsv(self, tmp_path, flux_points):
flux_points.write(tmp_path / "flux_points.ecsv")
actual = FluxPoints.read(tmp_path / "flux_points.ecsv")
assert str(flux_points) == str(actual)
def test_write_fits(self, tmp_path, flux_points):
flux_points.write(tmp_path / "tmp.fits")
actual = FluxPoints.read(tmp_path / "tmp.fits")
assert str(flux_points) == str(actual)
def flux_points_likelihood():
path = "$GAMMAPY_DATA/tests/spectrum/flux_points/binlike.fits"
return FluxPoints.read(path).to_sed_type("dnde")
def flux_points(request):
path = "$GAMMAPY_DATA/tests/spectrum/flux_points/" + request.param
return FluxPoints.read(path)
# In the Fermi-LAT catalogs, integral flux points are given. Currently the flux point fitter only works with differential flux points, so we apply the conversion here.
# In[ ]:
flux_points_3fgl = source_fermi_3fgl.flux_points.to_sed_type(
sed_type="dnde", model=source_fermi_3fgl.spectral_model())
flux_points_3fhl = source_fermi_3fhl.flux_points.to_sed_type(
sed_type="dnde", model=source_fermi_3fhl.spectral_model())
# Finally we stack the flux points into a single `~gammapy.spectrum.FluxPoints` object and drop the upper limit values, because currently we can't handle them in the fit:
# In[ ]:
# Stack flux point tables
flux_points = FluxPoints.stack(
[flux_points_gammacat, flux_points_3fhl, flux_points_3fgl])
t = flux_points.table
t["dnde_err"] = 0.5 * (t["dnde_errn"] + t["dnde_errp"])
# Remove upper limit points, where `dnde_errn = nan`
is_ul = np.isfinite(t["dnde_err"])
flux_points = FluxPoints(t[is_ul])
flux_points
# ## Power Law Fit
#
# First we start with fitting a simple `~gammapy.modeling.models.PowerLawSpectralModel`.
# In[ ]:
def run(self, e_ref, e_min, e_max, steps="all", atol="1e-6 s"):
"""Run light curve extraction.
Normalize integral and energy flux between emin and emax.
Parameters
----------
e_ref : `~astropy.units.Quantity`
reference energy of dnde flux normalization
e_min : `~astropy.units.Quantity`
minimum energy of integral and energy flux normalization interval
e_max : `~astropy.units.Quantity`
minimum energy of integral and energy flux normalization interval
steps : list of str
Which steps to execute. Available options are:
* "err": estimate symmetric error.
* "errn-errp": estimate asymmetric errors.
* "ul": estimate upper limits.
* "ts": estimate ts and sqrt(ts) values.
* "norm-scan": estimate fit statistic profiles.
By default all steps are executed.
atol : `~astropy.units.Quantity`
Tolerance value for time comparison with different scale. Default 1e-6 sec.
Returns
-------
lightcurve : `~gammapy.time.LightCurve`
the Light Curve object
"""
atol = u.Quantity(atol)
self.e_ref = e_ref
self.e_min = e_min
self.e_max = e_max
rows = []
self.group_table_info = group_datasets_in_time_interval(
datasets=self.datasets,
time_intervals=self.time_intervals,
atol=atol)
if np.all(self.group_table_info["Group_ID"] == -1):
raise ValueError(
"LightCurveEstimator: No datasets in time intervals")
for igroup, time_interval in enumerate(self.time_intervals):
index_dataset = np.where(
self.group_table_info["Group_ID"] == igroup)[0]
if len(index_dataset) == 0:
log.debug("No Dataset for the time interval " + str(igroup))
continue
row = {
"time_min": time_interval[0].mjd,
"time_max": time_interval[1].mjd
}
interval_list_dataset = Datasets(
[self.datasets[int(_)].copy() for _ in index_dataset])
self._set_scale_model(interval_list_dataset)
row.update(
self.estimate_time_bin_flux(interval_list_dataset,
time_interval, steps))
rows.append(row)
table = table_from_row_data(rows=rows, meta={"SED_TYPE": "likelihood"})
table = FluxPoints(table).to_sed_type("flux").table
return LightCurve(table)
# Fit norm in bands
diff_flux = list()
diff_flux_err = list()
e_err_hi = list()
e_err_lo = list()
energy = list()
for ii in range(len(binning) - 1):
energ = np.sqrt(binning[ii] * binning[ii + 1])
energy.append(energ)
e_err_hi.append(binning[ii + 1])
e_err_lo.append(binning[ii])
fit.fit_range = binning[[ii, ii + 1]]
fit.run()
res = fit.result[0].fit
diff_flux.append(res.model(energ).to('cm-2 s-1 TeV-1'))
err = res.model_with_uncertainties(energ.to('keV').value)
diff_flux_err.append(err.s * u.Unit('cm-2 s-1 keV-1'))
table = Table()
table['e_ref'] = energy
table['e_min'] = e_err_lo
table['e_max'] = e_err_hi
table['dnde'] = diff_flux
table['dnde_err'] = diff_flux_err
points = FluxPoints(table)
result = SpectrumResult(fit=best_fit, points=points)
result.plot_spectrum()
plt.savefig('fluxpoints.png')
1.23e-12,
8.37e-13,
])
e2dnde_err = np.array([
0.08e-11,
0.05e-11,
0.04e-11,
0.03e-11,
0.31e-12,
0.29e-12,
0.28e-12,
0.24e-12,
2.91e-13,
])
dnde = e2dnde / e_ref**2.0
dnde_err = dnde * e2dnde_err / e2dnde
table = Table()
table.meta["SED_TYPE"] = "dnde"
table["dnde"] = dnde
table["dnde"].unit = "cm-2 s-1 TeV-1"
table["dnde_err"] = dnde_err
table["dnde_err"].unit = "cm-2 s-1 TeV-1"
table["e_ref"] = e_ref
table["e_ref"].unit = "TeV"
flux_points_hawc = FluxPoints(table)
filename = Path("$GAMMAPY_EXTRA/datasets/hawc_crab/HAWC19_flux_points.fits")
flux_points_hawc.write(filename, overwrite=True)
# In the Fermi-LAT catalogs, integral flux points are given. Currently the flux point fitter only works with differential flux points, so we apply the conversion here.
# In[ ]:
flux_points_3fgl = source_fermi_3fgl.flux_points.to_sed_type(
sed_type="dnde", model=source_fermi_3fgl.spectral_model)
flux_points_3fhl = source_fermi_3fhl.flux_points.to_sed_type(
sed_type="dnde", model=source_fermi_3fhl.spectral_model)
# Finally we stack the flux points into a single `FluxPoints` object and drop the upper limit values, because currently we can't handle them in the fit:
# In[ ]:
# stack flux point tables
flux_points = FluxPoints.stack(
[flux_points_gammacat, flux_points_3fhl, flux_points_3fgl])
# drop the flux upper limit values
flux_points = flux_points.drop_ul()
# ## Power Law Fit
#
# First we start with fitting a simple [power law](https://docs.gammapy.org/0.11/api/gammapy.spectrum.models.PowerLaw.html#gammapy.spectrum.models.PowerLaw).
# In[ ]:
pwl = PowerLaw(index=2, amplitude="1e-12 cm-2 s-1 TeV-1", reference="1 TeV")
# After creating the model we run the fit by passing the `'flux_points'` and `'pwl'` objects:
# In[ ]:
datasets.append(dataset)
dataset_hess = Datasets(datasets).stack_reduce()
dataset_hess.name = "HESS"
dataset_hess.models = crab_model
# ### HAWC: 1D dataset for flux point fitting
#
# The HAWC flux point are taken from https://arxiv.org/pdf/1905.12518.pdf. Then these flux points are read from a pre-made FITS file and passed to a `FluxPointsDataset` together with the source spectral model.
#
# In[ ]:
# read flux points from https://arxiv.org/pdf/1905.12518.pdf
filename = "$GAMMAPY_DATA/hawc_crab/HAWC19_flux_points.fits"
flux_points_hawc = FluxPoints.read(filename)
dataset_hawc = FluxPointsDataset(crab_model, flux_points_hawc, name="HAWC")
# ## Datasets serialization
#
# The `datasets` object contains each dataset previously defined.
# It can be saved on disk as datasets.yaml, models.yaml, and several data files specific to each dataset. Then the `datasets` can be rebuild later from these files.
# In[ ]:
datasets = Datasets([dataset_fermi, dataset_hess, dataset_hawc])
path = Path("crab-3datasets")
path.mkdir(exist_ok=True)
datasets.write(path=path, prefix="crab_10GeV_100TeV", overwrite=True)
filedata = path / "crab_10GeV_100TeV_datasets.yaml"
