# ## Fit spectrum
#
# Now we'll fit a global model to the spectrum. First we do a joint likelihood fit to all observations. If you want to stack the observations see below. We will also produce a debug plot in order to show how the global fit matches one of the individual observations.
# In[ ]:
spectral_model = PowerLawSpectralModel(
index=2, amplitude=2e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.TeV
)
model = SkyModel(spectral_model=spectral_model)
for dataset in datasets:
dataset.models = model
fit_joint = Fit(datasets)
result_joint = fit_joint.run()
# we make a copy here to compare it later
model_best_joint = model.copy()
model_best_joint.spectral_model.parameters.covariance = (
result_joint.parameters.covariance
)
# In[ ]:
print(result_joint)
# Define map geometry
axis = MapAxis.from_edges(np.logspace(-1.0, 1.0, 10),
unit="TeV",
name="energy")
geom = WcsGeom.create(skydir=(0, 0),
binsz=0.02,
width=(2, 2),
frame="galactic",
axes=[axis])
# Define some observation parameters
# we are not simulating many pointings / observations
pointing = SkyCoord(0.2, 0.5, unit="deg", frame="galactic")
livetime = 20 * u.hour
exposure_map = make_map_exposure_true_energy(pointing=pointing,
livetime=livetime,
aeff=aeff,
geom=geom)
dataset = MapDataset(models=models, exposure=exposure_map)
npred = dataset.npred()
dataset.fake()
fit = Fit([dataset])
results = fit.run()
print(results)
print(models)
def test_map_fit(sky_model, geom, geom_etrue):
dataset_1 = get_map_dataset(sky_model,
geom,
geom_etrue,
evaluation_mode="local")
dataset_1.background_model.norm.value = 0.5
dataset_1.counts = dataset_1.npred()
dataset_2 = get_map_dataset(sky_model,
geom,
geom_etrue,
evaluation_mode="global")
dataset_2.counts = dataset_2.npred()
sky_model.parameters["sigma"].frozen = True
dataset_1.background_model.norm.value = 0.49
dataset_2.background_model.norm.value = 0.99
fit = Fit([dataset_1, dataset_2])
result = fit.run()
assert result.success
assert "minuit" in repr(result)
npred = dataset_1.npred().data.sum()
assert_allclose(npred, 6220.529956, rtol=1e-3)
assert_allclose(result.total_stat, 26008.040889, rtol=1e-3)
pars = result.parameters
assert_allclose(pars["lon_0"].value, 0.2, rtol=1e-2)
assert_allclose(pars.error("lon_0"), 0.002622, rtol=1e-2)
assert_allclose(pars["index"].value, 3, rtol=1e-2)
assert_allclose(pars.error("index"), 0.028967, rtol=1e-2)
assert_allclose(pars["amplitude"].value, 1e-11, rtol=1e-2)
assert_allclose(pars.error("amplitude"), 4.086756e-13, rtol=1e-2)
# background norm 1
assert_allclose(pars[8].value, 0.5, rtol=1e-2)
assert_allclose(pars.error(pars[8]), 0.0156, rtol=1e-2)
# background norm 2
assert_allclose(pars[11].value, 1, rtol=1e-2)
assert_allclose(pars.error(pars[11]), 0.02147, rtol=1e-2)
# test mask_safe evaluation
mask_safe = geom.energy_mask(emin=1 * u.TeV)
dataset_1.mask_safe = Map.from_geom(geom, data=mask_safe)
dataset_2.mask_safe = Map.from_geom(geom, data=mask_safe)
stat = fit.datasets.stat_sum()
assert_allclose(stat, 14447.196919)
# test model evaluation outside image
dataset_1.models[0].spatial_model.lon_0.value = 150
dataset_1.npred()
assert not dataset_1._evaluators[0].contributes
with mpl_plot_check():
dataset_1.plot_residuals()
def test_fit_pwl_sherpa(self, dataset):
fit = Fit(backend="sherpa", optimize_opts={"method": "simplex"})
result = fit.optimize(datasets=[dataset])
self.assert_result(result)
def test_map_fit(sky_model, geom, geom_etrue):
dataset_1 = get_map_dataset(sky_model,
geom,
geom_etrue,
evaluation_mode="local",
name="test-1")
dataset_1.background_model.norm.value = 0.5
dataset_1.counts = dataset_1.npred()
dataset_2 = get_map_dataset(sky_model,
geom,
geom_etrue,
evaluation_mode="global",
name="test-2")
dataset_2.counts = dataset_2.npred()
sky_model.parameters["sigma"].frozen = True
dataset_1.background_model.norm.value = 0.49
dataset_2.background_model.norm.value = 0.99
fit = Fit([dataset_1, dataset_2])
result = fit.run()
assert result.success
assert "minuit" in repr(result)
npred = dataset_1.npred().data.sum()
assert_allclose(npred, 7525.790688, rtol=1e-3)
assert_allclose(result.total_stat, 21700.253246, rtol=1e-3)
pars = result.parameters
assert_allclose(pars["lon_0"].value, 0.2, rtol=1e-2)
assert_allclose(pars["lon_0"].error, 0.002244, rtol=1e-2)
assert_allclose(pars["index"].value, 3, rtol=1e-2)
assert_allclose(pars["index"].error, 0.024277, rtol=1e-2)
assert_allclose(pars["amplitude"].value, 1e-11, rtol=1e-2)
assert_allclose(pars["amplitude"].error, 4.216154e-13, rtol=1e-2)
# background norm 1
assert_allclose(pars[8].value, 0.5, rtol=1e-2)
assert_allclose(pars[8].error, 0.015811, rtol=1e-2)
# background norm 2
assert_allclose(pars[11].value, 1, rtol=1e-2)
assert_allclose(pars[11].error, 0.02147, rtol=1e-2)
# test mask_safe evaluation
mask_safe = geom.energy_mask(emin=1 * u.TeV)
dataset_1.mask_safe = Map.from_geom(geom, data=mask_safe)
dataset_2.mask_safe = Map.from_geom(geom, data=mask_safe)
stat = fit.datasets.stat_sum()
assert_allclose(stat, 14824.282955)
# test model evaluation outside image
dataset_1.models[0].spatial_model.lon_0.value = 150
dataset_1.npred()
assert not dataset_1._evaluators[dataset_1.models[0]].contributes
region = sky_model.spatial_model.to_region()
with mpl_plot_check():
dataset_1.plot_residuals(region=region)
#
# First we start with fitting a simple `~gammapy.modeling.models.PowerLawSpectralModel`.
# In[ ]:
pwl = PowerLawSpectralModel(index=2,
amplitude="1e-12 cm-2 s-1 TeV-1",
reference="1 TeV")
model = SkyModel(spectral_model=pwl)
# After creating the model we run the fit by passing the `'flux_points'` and `'model'` objects:
# In[ ]:
dataset_pwl = FluxPointsDataset(model, flux_points)
fitter = Fit([dataset_pwl])
result_pwl = fitter.run()
# And print the result:
# In[ ]:
print(result_pwl)
# In[ ]:
print(pwl)
# Finally we plot the data points and the best fit model:
# In[ ]:
def fit(dataset):
return Fit([dataset])
# Define map geometry
axis = MapAxis.from_edges(np.logspace(-1.0, 1.0, 10),
unit="TeV",
name="energy")
geom = WcsGeom.create(skydir=(0, 0),
binsz=0.02,
width=(2, 2),
frame="galactic",
axes=[axis])
# Define some observation parameters
# we are not simulating many pointings / observations
pointing = SkyCoord(0.2, 0.5, unit="deg", frame="galactic")
livetime = 20 * u.hour
exposure_map = make_map_exposure_true_energy(pointing=pointing,
livetime=livetime,
aeff=aeff,
geom=geom)
dataset = MapDataset(models=models, exposure=exposure_map)
npred = dataset.npred()
dataset.fake()
fit = Fit()
results = fit.run([dataset])
print(results)
print(models)
def test_fov_bkg_maker_with_source_model(obs_dataset, exclusion_mask, caplog):
test_dataset = obs_dataset.copy(name="test-fov")
# crab model
spatial_model = PointSpatialModel(
lon_0="83.619deg", lat_0="22.024deg", frame="icrs"
)
spectral_model = PowerLawSpectralModel(
index=2.6, amplitude="4.5906e-11 cm-2 s-1 TeV-1", reference="1 TeV"
)
model = SkyModel(
spatial_model=spatial_model, spectral_model=spectral_model, name="test-source"
)
bkg_model = FoVBackgroundModel(dataset_name="test-fov")
test_dataset.models = [model, bkg_model]
# pre-fit both source and background to get reference model
Fit().run(test_dataset)
bkg_model_spec = test_dataset.models[f"{test_dataset.name}-bkg"].spectral_model
norm_ref = 0.897
assert not bkg_model_spec.norm.frozen
assert_allclose(bkg_model_spec.norm.value, norm_ref, rtol=1e-4)
assert_allclose(bkg_model_spec.tilt.value, 0.0, rtol=1e-4)
# apply scale method with pre-fitted source model and no exclusion_mask
bkg_model_spec.norm.value = 1
fov_bkg_maker = FoVBackgroundMaker(method="scale", exclusion_mask=None)
dataset = fov_bkg_maker.run(test_dataset)
bkg_model_spec = test_dataset.models[f"{dataset.name}-bkg"].spectral_model
assert_allclose(bkg_model_spec.norm.value, norm_ref, rtol=1e-4)
assert_allclose(bkg_model_spec.tilt.value, 0.0, rtol=1e-4)
# apply fit method with pre-fitted source model and no exlusion mask
bkg_model_spec.norm.value = 1
fov_bkg_maker = FoVBackgroundMaker(method="fit", exclusion_mask=None)
dataset = fov_bkg_maker.run(test_dataset)
bkg_model_spec = test_dataset.models[f"{dataset.name}-bkg"].spectral_model
assert_allclose(bkg_model_spec.norm.value, norm_ref, rtol=1e-4)
assert_allclose(bkg_model_spec.tilt.value, 0.0, rtol=1e-4)
# apply scale method with pre-fitted source model and exclusion_mask
bkg_model_spec.norm.value = 1
fov_bkg_maker = FoVBackgroundMaker(method="scale", exclusion_mask=exclusion_mask)
dataset = fov_bkg_maker.run(test_dataset)
bkg_model_spec = test_dataset.models[f"{dataset.name}-bkg"].spectral_model
assert_allclose(bkg_model_spec.norm.value, 0.830779, rtol=1e-4)
assert_allclose(bkg_model_spec.tilt.value, 0.0, rtol=1e-4)
# apply fit method with pre-fitted source model and exlusion mask
bkg_model_spec.norm.value = 1
fov_bkg_maker = FoVBackgroundMaker(method="fit", exclusion_mask=exclusion_mask)
dataset = fov_bkg_maker.run(test_dataset)
bkg_model_spec = test_dataset.models[f"{dataset.name}-bkg"].spectral_model
assert_allclose(bkg_model_spec.norm.value, 0.830779, rtol=1e-4)
assert_allclose(bkg_model_spec.tilt.value, 0.0, rtol=1e-4)
# Here we check that source parameters are correctly thawed after fit.
assert not dataset.models.parameters["index"].frozen
assert not dataset.models.parameters["lon_0"].frozen
# test
model.spectral_model.amplitude.value *= 1e5
fov_bkg_maker = FoVBackgroundMaker(method="scale")
dataset = fov_bkg_maker.run(test_dataset)
assert "WARNING" in [_.levelname for _ in caplog.records]
message1 = "FoVBackgroundMaker failed. Negative residuals counts for test-fov. Setting mask to False."
assert message1 in [_.message for _ in caplog.records]
def run_region(self, kr, lon, lat, radius):
# TODO: for now we have to read/create the allsky maps each in each job
# because we can't pickle <functools._lru_cache_wrapper object
# send this back to init when fixed
log.info(f"ROI {kr}: loading data")
# exposure
exposure_hpx = Map.read(
"$GAMMAPY_DATA/fermi_3fhl/fermi_3fhl_exposure_cube_hpx.fits.gz")
exposure_hpx.unit = "cm2 s"
# iem
iem_filepath = BASE_PATH / "data" / "gll_iem_v06_extrapolated.fits"
iem_fermi_extra = Map.read(iem_filepath)
# norm=1.1, tilt=0.03 see paper appendix A
model_iem = SkyModel(
PowerLawNormSpectralModel(norm=1.1, tilt=0.03),
TemplateSpatialModel(iem_fermi_extra, normalize=False),
name="iem_extrapolated",
)
# ROI
roi_time = time()
ROI_pos = SkyCoord(lon, lat, frame="galactic", unit="deg")
width = 2 * (radius + self.psf_margin)
# Counts
counts = Map.create(
skydir=ROI_pos,
width=width,
proj="CAR",
frame="galactic",
binsz=1 / 8.0,
axes=[self.energy_axis],
dtype=float,
)
counts.fill_by_coord({
"skycoord": self.events.radec,
"energy": self.events.energy
})
axis = MapAxis.from_nodes(counts.geom.axes[0].center,
name="energy_true",
unit="GeV",
interp="log")
wcs = counts.geom.wcs
geom = WcsGeom(wcs=wcs, npix=counts.geom.npix, axes=[axis])
coords = geom.get_coord()
# expo
data = exposure_hpx.interp_by_coord(coords)
exposure = WcsNDMap(geom, data, unit=exposure_hpx.unit, dtype=float)
# read PSF
psf_kernel = PSFKernel.from_table_psf(self.psf,
geom,
max_radius=self.psf_margin *
u.deg)
# Energy Dispersion
e_true = exposure.geom.axes[0].edges
e_reco = counts.geom.axes[0].edges
edisp = EDispKernel.from_diagonal_response(e_true=e_true,
e_reco=e_reco)
# fit mask
if coords["lon"].min() < 90 * u.deg and coords["lon"].max(
) > 270 * u.deg:
coords["lon"][coords["lon"].value > 180] -= 360 * u.deg
mask = (
(coords["lon"] >= coords["lon"].min() + self.psf_margin * u.deg)
& (coords["lon"] <= coords["lon"].max() - self.psf_margin * u.deg)
& (coords["lat"] >= coords["lat"].min() + self.psf_margin * u.deg)
& (coords["lat"] <= coords["lat"].max() - self.psf_margin * u.deg))
mask_fermi = WcsNDMap(counts.geom, mask)
log.info(f"ROI {kr}: pre-computing diffuse")
# IEM
eval_iem = MapEvaluator(model=model_iem,
exposure=exposure,
psf=psf_kernel,
edisp=edisp)
bkg_iem = eval_iem.compute_npred()
# ISO
eval_iso = MapEvaluator(model=self.model_iso,
exposure=exposure,
edisp=edisp)
bkg_iso = eval_iso.compute_npred()
# merge iem and iso, only one local normalization is fitted
dataset_name = "3FHL_ROI_num" + str(kr)
background_total = bkg_iem + bkg_iso
background_model = BackgroundModel(background_total,
name="bkg_iem+iso",
datasets_names=[dataset_name])
background_model.parameters["norm"].min = 0.0
# Sources model
in_roi = self.FHL3.positions.galactic.contained_by(wcs)
FHL3_roi = []
for ks in range(len(self.FHL3.table)):
if in_roi[ks] == True:
model = self.FHL3[ks].sky_model()
model.spatial_model.parameters.freeze_all() # freeze spatial
model.spectral_model.parameters["amplitude"].min = 0.0
if isinstance(model.spectral_model, PowerLawSpectralModel):
model.spectral_model.parameters["index"].min = 0.1
model.spectral_model.parameters["index"].max = 10.0
else:
model.spectral_model.parameters["alpha"].min = 0.1
model.spectral_model.parameters["alpha"].max = 10.0
FHL3_roi.append(model)
model_total = Models([background_model] + FHL3_roi)
# Dataset
dataset = MapDataset(
models=model_total,
counts=counts,
exposure=exposure,
psf=psf_kernel,
edisp=edisp,
mask_fit=mask_fermi,
name=dataset_name,
)
cat_stat = dataset.stat_sum()
datasets = Datasets([dataset])
log.info(f"ROI {kr}: running fit")
fit = Fit(datasets)
results = fit.run(**self.optimize_opts)
print("ROI_num", str(kr), "\n", results)
fit_stat = datasets.stat_sum()
if results.message != "Optimization failed.":
datasets.write(path=Path(self.resdir),
prefix=dataset.name,
overwrite=True)
np.savez(
self.resdir / f"3FHL_ROI_num{kr}_fit_infos.npz",
message=results.message,
stat=[cat_stat, fit_stat],
)
exec_time = time() - roi_time
print("ROI", kr, " time (s): ", exec_time)
log.info(f"ROI {kr}: running flux points")
for model in FHL3_roi:
if (self.FHL3[model.name].data["ROI_num"] == kr
and self.FHL3[model.name].data["Signif_Avg"] >=
self.sig_cut):
flux_points = FluxPointsEstimator(
e_edges=self.El_flux,
source=model.name,
n_sigma_ul=2,
).run(datasets=datasets)
filename = self.resdir / f"{model.name}_flux_points.fits"
flux_points.write(filename, overwrite=True)
exec_time = time() - roi_time - exec_time
print("ROI", kr, " Flux points time (s): ", exec_time)
def fit(dataset):
return Fit([dataset], backend="minuit")
def run_analysis_1d(target_dict, e_reco, fluxp_edges, debug):
"""Run joint spectral analysis for the selected target"""
tag = target_dict["tag"]
name = target_dict["name"]
log.info(f"Running 1d analysis, {tag}")
path_res = Path(tag + "/results/")
ra = target_dict["ra"]
dec = target_dict["dec"]
on_size = target_dict["on_size"]
e_decorr = target_dict["e_decorr"]
target_pos = SkyCoord(ra, dec, frame="icrs")
on_radius = Angle(on_size)
log.info(f"Running observations selection")
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
mask = data_store.obs_table["TARGET_NAME"] == name
obs_table = data_store.obs_table[mask]
observations = data_store.get_observations(obs_table["OBS_ID"])
if debug is True:
observations = [observations[0]]
log.info(f"Running data reduction")
# Reflected regions background estimation
on_region = CircleSkyRegion(center=target_pos, radius=on_radius)
dataset_maker = SpectrumDatasetMaker(region=on_region,
e_reco=e_reco,
e_true=e_reco,
containment_correction=True)
bkg_maker = ReflectedRegionsBackgroundMaker()
safe_mask_masker = SafeMaskMaker(methods=["edisp-bias"], bias_percent=10)
datasets = []
for observation in observations:
dataset = dataset_maker.run(observation,
selection=["counts", "aeff", "edisp"])
dataset_on_off = bkg_maker.run(dataset, observation)
dataset_on_off = safe_mask_masker.run(dataset_on_off, observation)
datasets.append(dataset_on_off)
log.info(f"Running fit ...")
model = PowerLawSpectralModel(index=2,
amplitude=2e-11 * u.Unit("cm-2 s-1 TeV-1"),
reference=e_decorr)
for dataset in datasets:
dataset.models = SkyModel(spectral_model=model)
fit_joint = Fit(datasets)
result_joint = fit_joint.run()
parameters = model.parameters
parameters.covariance = result_joint.parameters.covariance
log.info(f"Writing {path_res}")
write_fit_summary(parameters,
str(path_res / "results-summary-fit-1d.yaml"))
log.info(f"Running flux points estimation")
fpe = FluxPointsEstimator(datasets=datasets, e_edges=fluxp_edges)
flux_points = fpe.run()
flux_points.table["is_ul"] = flux_points.table["ts"] < 4
keys = [
"e_ref",
"e_min",
"e_max",
"dnde",
"dnde_errp",
"dnde_errn",
"is_ul",
"dnde_ul",
]
log.info(f"Writing {path_res}")
flux_points.table_formatted[keys].write(path_res / "flux-points-1d.ecsv",
format="ascii.ecsv")
def run_analysis_3d(target_dict, fluxp_edges, debug):
"""Run stacked 3D analysis for the selected target.
Notice that, for the sake of time saving, we run a stacked analysis, as opposed
to the joint analysis that is performed in the reference paper.
"""
tag = target_dict["tag"]
log.info(f"running 3d analysis, {tag}")
path_res = Path(tag + "/results/")
txt = Path("config_template.yaml").read_text()
txt = txt.format_map(target_dict)
config = AnalysisConfig.from_yaml(txt)
log.info(f"Running observations selection")
analysis = Analysis(config)
analysis.get_observations()
log.info(f"Running data reduction")
analysis.get_datasets()
# TODO: Improve safe mask handling in Analysis. the mask should be applied run-by-run
maker_safe_mask = SafeMaskMaker(methods=["edisp-bias", "bkg-peak"])
stacked = maker_safe_mask.run(analysis.datasets[0])
log.info(f"Running fit ...")
ra = target_dict["ra"]
dec = target_dict["dec"]
e_decorr = target_dict["e_decorr"]
spectral_model = Model.create("PowerLawSpectralModel", reference=e_decorr)
spatial_model = Model.create(target_dict["spatial_model"],
lon_0=ra,
lat_0=dec)
if target_dict["spatial_model"] == "DiskSpatialModel":
spatial_model.e.frozen = False
sky_model = SkyModel(spatial_model=spatial_model,
spectral_model=spectral_model,
name=tag)
stacked.models = sky_model
stacked.background_model.norm.frozen = False
fit = Fit([stacked])
result = fit.run()
parameters = stacked.models.parameters
model_npars = len(sky_model.parameters.names)
parameters.covariance = result.parameters.covariance[0:model_npars,
0:model_npars]
log.info(f"Writing {path_res}")
write_fit_summary(parameters,
str(path_res / "results-summary-fit-3d.yaml"))
log.info("Running flux points estimation")
# TODO: This is a workaround to re-optimize the bkg. Remove it once it's added to the Analysis class
for par in stacked.parameters:
if par is not stacked.background_model.norm:
par.frozen = True
reoptimize = True if debug is False else False
fpe = FluxPointsEstimator(datasets=[stacked],
e_edges=fluxp_edges,
source=tag,
reoptimize=reoptimize)
flux_points = fpe.run()
flux_points.table["is_ul"] = flux_points.table["ts"] < 4
keys = [
"e_ref",
"e_min",
"e_max",
"dnde",
"dnde_errp",
"dnde_errn",
"is_ul",
"dnde_ul",
]
log.info(f"Writing {path_res}")
flux_points.table_formatted[keys].write(path_res / "flux-points-3d.ecsv",
format="ascii.ecsv")
def datasets(self, datasets):
self._datasets = self._check_datasets(datasets)
self.fit = Fit(datasets)
self.fit_result = None
def data_fit(stacked):
# Data fitting
fit = Fit(stacked)
result = fit.run(optimize_opts={"print_level": 1})
def test_map_fit(sky_model, geom, geom_etrue):
dataset_1 = get_map_dataset(geom, geom_etrue, name="test-1")
dataset_2 = get_map_dataset(geom, geom_etrue, name="test-2")
datasets = Datasets([dataset_1, dataset_2])
models = Models(datasets.models)
models.insert(0, sky_model)
models["test-1-bkg"].spectral_model.norm.value = 0.5
models["test-model"].spatial_model.sigma.frozen = True
datasets.models = models
dataset_2.counts = dataset_2.npred()
dataset_1.counts = dataset_1.npred()
models["test-1-bkg"].spectral_model.norm.value = 0.49
models["test-2-bkg"].spectral_model.norm.value = 0.99
fit = Fit(datasets)
result = fit.run()
assert result.success
assert "minuit" in repr(result)
npred = dataset_1.npred().data.sum()
assert_allclose(npred, 7525.790688, rtol=1e-3)
assert_allclose(result.total_stat, 21659.2139, rtol=1e-3)
pars = result.parameters
assert_allclose(pars["lon_0"].value, 0.2, rtol=1e-2)
assert_allclose(pars["lon_0"].error, 0.002244, rtol=1e-2)
assert_allclose(pars["index"].value, 3, rtol=1e-2)
assert_allclose(pars["index"].error, 0.024277, rtol=1e-2)
assert_allclose(pars["amplitude"].value, 1e-11, rtol=1e-2)
assert_allclose(pars["amplitude"].error, 4.216154e-13, rtol=1e-2)
# background norm 1
assert_allclose(pars[8].value, 0.5, rtol=1e-2)
assert_allclose(pars[8].error, 0.015811, rtol=1e-2)
# background norm 2
assert_allclose(pars[11].value, 1, rtol=1e-2)
assert_allclose(pars[11].error, 0.02147, rtol=1e-2)
# test mask_safe evaluation
mask_safe = geom.energy_mask(energy_min=1 * u.TeV)
dataset_1.mask_safe = Map.from_geom(geom, data=mask_safe)
dataset_2.mask_safe = Map.from_geom(geom, data=mask_safe)
stat = fit.datasets.stat_sum()
assert_allclose(stat, 14823.579908, rtol=1e-5)
region = sky_model.spatial_model.to_region()
initial_counts = dataset_1.counts.copy()
with mpl_plot_check():
dataset_1.plot_residuals(kwargs_spectral=dict(region=region))
# check dataset has not changed
assert initial_counts == dataset_1.counts
# test model evaluation outside image
dataset_1.models[0].spatial_model.lon_0.value = 150
dataset_1.npred()
assert not dataset_1._evaluators["test-model"].contributes
def test_datasets_to_io(tmp_path):
filedata = "$GAMMAPY_DATA/tests/models/gc_example_datasets.yaml"
filemodel = "$GAMMAPY_DATA/tests/models/gc_example_models.yaml"
datasets = Datasets.read(filedata, filemodel)
assert len(datasets) == 2
dataset0 = datasets[0]
assert dataset0.name == "gc"
assert dataset0.counts.data.sum() == 6824
assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1)
assert dataset0.psf is not None
assert dataset0.edisp is not None
assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1)
assert dataset0.background_model.name == "background_irf_gc"
dataset1 = datasets[1]
assert dataset1.name == "g09"
assert dataset1.background_model.name == "background_irf_g09"
assert (
dataset0.models["gll_iem_v06_cutout"] == dataset1.models["gll_iem_v06_cutout"]
)
assert isinstance(dataset0.models, Models)
assert len(dataset0.models) == 5
assert dataset0.models[0].name == "gc"
assert dataset0.models[1].name == "gll_iem_v06_cutout"
assert (
dataset0.models["background_irf_gc"].parameters["norm"]
is dataset1.models["background_irf_g09"].parameters["norm"]
)
assert (
dataset0.models["gc"].parameters["reference"]
is dataset1.models["g09"].parameters["reference"]
)
assert_allclose(dataset1.models["g09"].parameters["lon_0"].value, 0.9, atol=0.1)
datasets.write(tmp_path, prefix="written")
datasets_read = Datasets.read(
tmp_path / "written_datasets.yaml", tmp_path / "written_models.yaml"
)
assert len(datasets.parameters) == 21
assert len(datasets_read) == 2
dataset0 = datasets_read[0]
assert dataset0.counts.data.sum() == 6824
assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1)
assert dataset0.psf is not None
assert dataset0.edisp is not None
assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1)
Fit(datasets).run()
assert_allclose(
datasets.models["background_irf_g09"].covariance,
datasets.models["background_irf_gc"].covariance,
)
# In[ ]:
pwl = PowerLawSpectralModel(
index=2, amplitude="1e-12 cm-2 s-1 TeV-1", reference="1 TeV"
)
# After creating the model we run the fit by passing the `'flux_points'` and `'pwl'` objects:
# In[ ]:
dataset_pwl = FluxPointsDataset(pwl, flux_points, likelihood="chi2assym")
fitter = Fit(dataset_pwl)
result_pwl = fitter.run()
# And print the result:
# In[ ]:
print(result_pwl)
# In[ ]:
print(pwl)
def data_fit(datasets):
fit = Fit(datasets)
result = fit.run()
def data_fit(datasets):
fit = Fit(datasets)
result = fit.run(optimize_opts={"print_level": 1})
def test_fit_pwl_minuit(self, dataset):
fit = Fit()
result = fit.run(dataset)
self.assert_result(result)
fig, ax, cbar = npred.sum_over_axes().plot(add_cbar=True)
ax.scatter(
[lon_0_1, lon_0_2, pointing.galactic.l.degree],
[lat_0_1, lat_0_2, pointing.galactic.b.degree],
transform=ax.get_transform("galactic"),
marker="+",
color="cyan",
)
# plt.show()
plt.clf()
dataset.fake()
dataset.counts.sum_over_axes().plot()
# plt.show()
plt.clf()
models.parameters.set_error(spatial_model_1.lon_0, 0.1 * u.deg)
models.parameters.set_error(spatial_model_1.lat_0, 0.1 * u.deg)
models.parameters.set_error(spatial_model_2.lon_0, 0.1 * u.deg)
models.parameters.set_error(spatial_model_1.lat_0, 0.1 * u.deg)
models.parameters.set_error(spectral_model_1.amplitude,
1e-12 * u.Unit("cm-2 s-1 TeV-1"))
models.parameters.set_error(spectral_model_2.amplitude,
1e-12 * u.Unit("cm-2 s-1 TeV-1"))
fit = Fit(dataset)
fit.run()
def run(self,
datasets,
parameter,
steps="all",
null_value=1e-150,
scan_values=None):
"""Run the parameter estimator.
Parameters
----------
datasets : `~gammapy.datasets.Datasets`
The datasets used to estimate the model parameter
parameter : `~gammapy.modeling.Parameter`
the parameter to be estimated
steps : list of str
Which steps to execute. Available options are:
* "err": estimate symmetric error from covariance
* "ts": estimate delta TS with parameter null (reference) value
* "errn-errp": estimate asymmetric errors.
* "ul": estimate upper limits.
* "scan": estimate fit statistic profiles.
By default all steps are executed.
null_value : float
the null value to be used for delta TS estimation.
Default is 1e-150 since 0 can be an issue for some parameters.
scan_values : `numpy.ndarray`
Array of parameter values to be used for the fit statistic profile.
If set to None, scan values are automatically calculated. Default is None.
Returns
-------
result : dict
Dict with the various parameter estimation values.
"""
self.datasets = self._check_datasets(datasets)
self.fit = Fit(datasets)
self.fit_result = None
with self.datasets.parameters.restore_values:
if not self.reoptimize:
self._freeze_parameters(parameter)
if steps == "all":
steps = ["err", "ts", "errp-errn", "ul", "scan"]
result = self._find_best_fit(parameter)
TS1 = result["stat"]
value_max = result[parameter.name]
if "err" in steps:
res = self.fit.covariance()
value_err = res.parameters[parameter].error
result.update({f"{parameter.name}_err": value_err})
if "errp-errn" in steps:
res = self.fit.confidence(parameter=parameter,
sigma=self.sigma)
result.update({
f"{parameter.name}_errp": res["errp"],
f"{parameter.name}_errn": res["errn"],
})
if "ul" in steps:
res = self.fit.confidence(parameter=parameter,
sigma=self.sigma_ul)
result.update(
{f"{parameter.name}_ul": res["errp"] + value_max})
if "ts" in steps:
TS0 = self._estimate_ts_for_null_value(parameter, null_value)
res = TS0 - TS1
result.update({
"sqrt_ts": np.sqrt(res),
"ts": res,
"null_value": null_value
})
# TODO: should not need this
self.fit.optimize()
if "scan" in steps:
if scan_values is None:
scan_values = self._compute_scan_values(
value_max, value_err, parameter.min, parameter.max)
res = self.fit.stat_profile(parameter,
values=scan_values,
reoptimize=self.reoptimize)
result.update({
f"{parameter.name}_scan": res["values"],
"stat_scan": res["stat"]
})
return result
def run_region(self, kr, lon, lat, radius):
# TODO: for now we have to read/create the allsky maps each in each job
# because we can't pickle <functools._lru_cache_wrapper object
# send this back to init when fixed
# exposure
exposure_hpx = Map.read(
self.datadir + "/fermi_3fhl/fermi_3fhl_exposure_cube_hpx.fits.gz")
exposure_hpx.unit = "cm2 s"
# background iem
infile = self.datadir + "/catalogs/fermi/gll_iem_v06.fits.gz"
outfile = self.resdir + "/gll_iem_v06_extra.fits"
model_iem = extrapolate_iem(infile, outfile, self.logEc_extra)
# ROI
roi_time = time()
ROI_pos = SkyCoord(lon, lat, frame="galactic", unit="deg")
width = 2 * (radius + self.psf_margin)
# Counts
counts = Map.create(
skydir=ROI_pos,
width=width,
proj="CAR",
coordsys="GAL",
binsz=self.dlb,
axes=[self.energy_axis],
dtype=float,
)
counts.fill_by_coord({
"skycoord": self.events.radec,
"energy": self.events.energy
})
axis = MapAxis.from_nodes(counts.geom.axes[0].center,
name="energy",
unit="GeV",
interp="log")
wcs = counts.geom.wcs
geom = WcsGeom(wcs=wcs, npix=counts.geom.npix, axes=[axis])
coords = counts.geom.get_coord()
# expo
data = exposure_hpx.interp_by_coord(coords)
exposure = WcsNDMap(geom, data, unit=exposure_hpx.unit, dtype=float)
# read PSF
psf_kernel = PSFKernel.from_table_psf(self.psf,
counts.geom,
max_radius=self.psf_margin *
u.deg)
# Energy Dispersion
e_true = exposure.geom.axes[0].edges
e_reco = counts.geom.axes[0].edges
edisp = EnergyDispersion.from_diagonal_response(e_true=e_true,
e_reco=e_reco)
# fit mask
if coords["lon"].min() < 90 * u.deg and coords["lon"].max(
) > 270 * u.deg:
coords["lon"][coords["lon"].value > 180] -= 360 * u.deg
mask = (
(coords["lon"] >= coords["lon"].min() + self.psf_margin * u.deg)
& (coords["lon"] <= coords["lon"].max() - self.psf_margin * u.deg)
& (coords["lat"] >= coords["lat"].min() + self.psf_margin * u.deg)
& (coords["lat"] <= coords["lat"].max() - self.psf_margin * u.deg))
mask_fermi = WcsNDMap(counts.geom, mask)
# IEM
eval_iem = MapEvaluator(model=model_iem,
exposure=exposure,
psf=psf_kernel,
edisp=edisp)
bkg_iem = eval_iem.compute_npred()
# ISO
eval_iso = MapEvaluator(model=self.model_iso,
exposure=exposure,
edisp=edisp)
bkg_iso = eval_iso.compute_npred()
# merge iem and iso, only one local normalization is fitted
background_total = bkg_iem + bkg_iso
background_model = BackgroundModel(background_total)
background_model.parameters["norm"].min = 0.0
# Sources model
in_roi = self.FHL3.positions.galactic.contained_by(wcs)
FHL3_roi = []
for ks in range(len(self.FHL3.table)):
if in_roi[ks] == True:
model = self.FHL3[ks].sky_model()
model.spatial_model.parameters.freeze_all() # freeze spatial
model.spectral_model.parameters["amplitude"].min = 0.0
if isinstance(model.spectral_model, PowerLawSpectralModel):
model.spectral_model.parameters["index"].min = 0.1
model.spectral_model.parameters["index"].max = 10.0
else:
model.spectral_model.parameters["alpha"].min = 0.1
model.spectral_model.parameters["alpha"].max = 10.0
FHL3_roi.append(model)
model_total = SkyModels(FHL3_roi)
# Dataset
dataset = MapDataset(
model=model_total,
counts=counts,
exposure=exposure,
psf=psf_kernel,
edisp=edisp,
background_model=background_model,
mask_fit=mask_fermi,
name="3FHL_ROI_num" + str(kr),
)
cat_stat = dataset.stat_sum()
datasets = Datasets([dataset])
fit = Fit(datasets)
results = fit.run(optimize_opts=self.optimize_opts)
print("ROI_num", str(kr), "\n", results)
fit_stat = datasets.stat_sum()
if results.message == "Optimization failed.":
pass
else:
datasets.to_yaml(path=Path(self.resdir),
prefix=dataset.name,
overwrite=True)
np.save(
self.resdir + "/3FHL_ROI_num" + str(kr) + "_covariance.npy",
results.parameters.covariance,
)
np.savez(
self.resdir + "/3FHL_ROI_num" + str(kr) + "_fit_infos.npz",
message=results.message,
stat=[cat_stat, fit_stat],
)
exec_time = time() - roi_time
print("ROI", kr, " time (s): ", exec_time)
for model in FHL3_roi:
if (self.FHL3[model.name].data["ROI_num"] == kr
and self.FHL3[model.name].data["Signif_Avg"] >=
self.sig_cut):
flux_points = FluxPointsEstimator(
datasets=datasets,
e_edges=self.El_flux,
source=model.name,
sigma_ul=2.0,
).run()
filename = self.resdir + "/" + model.name + "_flux_points.fits"
flux_points.write(filename, overwrite=True)
exec_time = time() - roi_time - exec_time
print("ROI", kr, " Flux points time (s): ", exec_time)
def _setup_fit(self, datasets):
# TODO: make fit stateless and configurable
if self._fit is None or datasets is not self._fit.datasets:
self._fit = Fit(datasets)
def data_fit(stacked):
# Data fitting
fit = Fit(optimize_opts={"print_level": 1})
result = fit.run(datasets=stacked)
print(result.success)
def estimate_time_bin_flux(self, datasets, time_interval, steps="all"):
"""Estimate flux point for a single energy group.
Parameters
----------
datasets : `~gammapy.modeling.Datasets`
the list of dataset object
time_interval : astropy.time.Time`
Start and stop time for each interval
steps : list of str
Which steps to execute. Available options are:
* "err": estimate symmetric error.
* "errn-errp": estimate asymmetric errors.
* "ul": estimate upper limits.
* "ts": estimate ts and sqrt(ts) values.
* "norm-scan": estimate likelihood profiles.
By default all steps are executed.
Returns
-------
result : dict
Dict with results for the flux point.
"""
self.fit = Fit(datasets)
result = {
"e_ref": self.e_ref,
"e_min": self.e_min,
"e_max": self.e_max,
"ref_dnde": self.ref_model(self.e_ref),
"ref_flux": self.ref_model.integral(self.e_min, self.e_max),
"ref_eflux": self.ref_model.energy_flux(self.e_min, self.e_max),
"ref_e2dnde": self.ref_model(self.e_ref) * self.e_ref**2,
}
result.update(self.estimate_norm())
if not result.pop("success"):
log.warning("Fit failed for time bin between {t_min} and {t_max},"
" setting NaN.".format(t_min=time_interval[0].mjd,
t_max=time_interval[1].mjd))
if steps == "all":
steps = ["err", "counts", "errp-errn", "ul", "ts", "norm-scan"]
if "err" in steps:
result.update(self.estimate_norm_err())
if "counts" in steps:
result.update(self.estimate_counts(datasets))
if "ul" in steps:
result.update(self.estimate_norm_ul(datasets))
if "errp-errn" in steps:
result.update(self.estimate_norm_errn_errp())
if "ts" in steps:
result.update(self.estimate_norm_ts(datasets))
if "norm-scan" in steps:
result.update(self.estimate_norm_scan())
return result
def data_fit(stacked):
fit = Fit([stacked])
result = fit.run(optimize_opts={"print_level": 1})
def __init__(self, energy_edges=[1, 10] * u.TeV, **kwargs):
self.energy_edges = energy_edges
fit = Fit(confidence_opts={"backend": "scipy"})
kwargs.setdefault("fit", fit)
super().__init__(**kwargs)
# Now we'll fit a model to the spectrum with the `Fit` class. First we load a power law model with an initial value for the index and the amplitude and then wo do a likelihood fit. The fit results are printed below.
# In[ ]:
model = PowerLawSpectralModel(index=4,
amplitude="1.3e-9 cm-2 s-1 TeV-1",
reference="0.02 TeV")
emin_fit, emax_fit = (0.04 * u.TeV, 0.4 * u.TeV)
for obs in extraction.spectrum_observations:
obs.model = model
obs.mask_fit = obs.counts.energy_mask(emin=emin_fit, emax=emax_fit)
joint_fit = Fit(extraction.spectrum_observations)
joint_result = joint_fit.run()
model.parameters.covariance = joint_result.parameters.covariance
print(joint_result)
# Now you might want to do the stacking here even if in our case there is only one observation which makes it superfluous.
# We can compute flux points by fitting the norm of the global model in energy bands.
# In[ ]:
e_edges = np.logspace(np.log10(0.04), np.log10(0.4), 7) * u.TeV
dataset = Datasets(extraction.spectrum_observations).stack_reduce()
dataset.model = model
