def load_data(input_dir, dataset_config, exclusion_map=None):
on_region_radius = dataset_config['on_radius']
e_reco_bins = dataset_config['e_reco_bins']
e_true_bins = dataset_config['e_true_bins']
containment = dataset_config['containment_correction']
ds = DataStore.from_dir(input_dir)
observations = ds.get_observations(ds.obs_table['OBS_ID'].data)
# observations = ds.get_observations(ds.hdu_table['OBS_ID'].data) # this is plenty wrong
source_position = dataset_config['source_position']
on_region = CircleSkyRegion(center=source_position,
radius=on_region_radius)
print('Estimating Background')
bkg_estimate = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_map)
bkg_estimate.run()
print('Extracting Count Spectra')
extract = SpectrumExtraction(
observations=observations,
bkg_estimate=bkg_estimate.result,
e_true=e_true_bins,
e_reco=e_reco_bins,
containment_correction=containment,
use_recommended_erange=False,
)
extract.run()
if dataset_config['stack']:
return [extract.spectrum_observations.stack()]
else:
return extract.spectrum_observations
def create_data(input_dir, dataset_config, exclusion_map=None):
telescope = dataset_config['telescope']
on_region_radius = dataset_config['on_radius']
energy_bins = dataset_config['e_reco_bins']
containment = dataset_config['containment_correction']
ds = DataStore.from_dir(os.path.join(input_dir, telescope))
observations = ds.get_observations(ds.obs_table['OBS_ID'].data)
t_obs = sum([o.observation_live_time_duration for o in observations])
# from IPython import embed; embed()
print(f'Total obstime for {telescope} is {t_obs.to("h")}')
source_position = dataset_config['source_position']
on_region = CircleSkyRegion(center=source_position,
radius=on_region_radius)
print('Estimating Background')
bkg_estimate = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_map)
bkg_estimate.run()
print('Extracting Count Spectra')
extract = SpectrumExtraction(
observations=observations,
bkg_estimate=bkg_estimate.result,
e_true=energy_bins,
e_reco=energy_bins,
containment_correction=containment,
use_recommended_erange=False, # TODO this might have to be checked.
)
extract.run()
return extract
def bkg_estimate(observations, on_region, exclusion_mask):
"""An example background estimate"""
est = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_mask,
min_distance_input="0.2 deg",
)
est.run()
return est.result
def bkg_estimator(observations, exclusion_mask, on_region):
"""Example background estimator for testing."""
maker = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_mask,
min_distance_input="0.2 deg",
)
maker.run()
return maker
def stats_stacked_bad_on_region(bad_on_region, observations):
bge = ReflectedRegionsBackgroundEstimator(
on_region=bad_on_region, observations=observations
)
bge.run()
return ObservationStats.stack(
[
ObservationStats.from_observation(obs, bg)
for obs, bg in zip(observations, bge.result)
]
)
def run_extraction(self):
"""Run all steps for the spectrum extraction."""
self.background_estimator = ReflectedRegionsBackgroundEstimator(
observations=self.observations, **self.config["background"])
self.background_estimator.run()
self.extraction = SpectrumExtraction(
observations=self.observations,
bkg_estimate=self.background_estimator.result,
**self.config["extraction"])
self.extraction.run()
def setup_class(cls):
datastore = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
obs_ids = [23523, 23526]
on_region = CircleSkyRegion(
SkyCoord(83.63 * u.deg, 22.01 * u.deg, frame="icrs"), 0.3 * u.deg)
obs_stats_list = []
for obs_id in obs_ids:
obs = datastore.obs(obs_id)
bkg = ReflectedRegionsBackgroundEstimator(on_region=on_region,
observations=[obs])
bkg.run()
bg_estimate = bkg.result[0]
obs_stats = ObservationStats.from_observation(obs, bg_estimate)
obs_stats_list.append(obs_stats)
cls.obs_summary = ObservationSummary(obs_stats_list)
def test_extract_cta_1dc_data(caplog):
datastore = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/")
obs_ids = [110380, 111140]
observations = datastore.get_observations(obs_ids)
pos = SkyCoord(0.0, 0.0, unit="deg", frame="galactic")
radius = Angle(0.11, "deg")
on_region = CircleSkyRegion(pos, radius)
est = ReflectedRegionsBackgroundEstimator(observations=observations,
on_region=on_region,
min_distance_input="0.2 deg")
est.run()
# This will test non PSF3D input as well as absence of default thresholds
extract = SpectrumExtraction(bkg_estimate=est.result,
observations=observations,
containment_correction=True)
extract.run()
extract.compute_energy_threshold(method_lo="area_max", area_percent_lo=10)
actual = extract.spectrum_observations[0].energy_range[0]
assert_quantity_allclose(actual, 0.774263 * u.TeV, rtol=1e-3)
def extract_spectra_iact(dataset):
"""Extract 1d spectra for IACT dataset"""
log.info(f"Extracting 1d spectra for {dataset.name} dataset")
datastore = DataStore.from_dir(f"data/{dataset.name}")
observations = datastore.get_observations(dataset.obs_ids)
on_region = CircleSkyRegion(center=config.source_pos,
radius=dataset.on_radius)
exclusion_mask = config.get_exclusion_mask()
bkg_estimate = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_mask)
bkg_estimate.run()
extract = SpectrumExtraction(
observations=observations,
bkg_estimate=bkg_estimate.result,
e_true=config.energy_bins,
e_reco=config.energy_bins,
containment_correction=dataset.containment_correction,
)
extract.run()
path = f"results/spectra/{dataset.name}/"
log.info(f"Writing to {path}")
if dataset.name == "fact":
# For FACT the IRFs are the same for all observations
# So we only store a stacked spectrum and response
# plus we add a LO_THRESHOLD keyword was missing
obs = extract.spectrum_observations.stack()
obs.lo_threshold = 0.4 * u.TeV
# we are writing a single observation, as for Fermi
obs.write(path, use_sherpa=True, overwrite=True)
else:
extract.write(path, ogipdir="", use_sherpa=True, overwrite=True)
def extract_spectra_IACT(dataset):
"""Extract 1d spectra for IACT dataset"""
log.info(f"Extracting 1d spectra for {dataset.name} dataset")
# Dataset class has already the method to obtain the gammapy DataStore object
datastore = dataset.get_DataStore()
obs_list = datastore.obs_list(dataset.obs_ids)
on_region = CircleSkyRegion(center=dataset.source_pos,
radius=dataset.on_radius)
exclusion_mask = config.get_exclusion_mask()
bkg_estimate = ReflectedRegionsBackgroundEstimator(
obs_list=obs_list, on_region=on_region, exclusion_mask=exclusion_mask)
bkg_estimate.run()
extract = SpectrumExtraction(
obs_list=obs_list,
bkg_estimate=bkg_estimate.result,
containment_correction=dataset.containment_correction,
)
extract.run()
path = f"{config.repo_path}/results/spectra/{dataset.name}/"
log.info(f"Writing to {path}")
if dataset.name == "fact":
# For FACT the IRFs are the same for all observations
# So we only store a stacked spectrum and response
# plus we add a LO_THRESHOLD keyword was missing
obs = extract.observations.stack()
obs.lo_threshold = 0.4 * u.TeV
# we are writing a single observation, as for Fermi
obs.write(path, use_sherpa=True, overwrite=True)
else:
extract.write(path, ogipdir="", use_sherpa=True, overwrite=True)
def stats_bad_on_region(bad_on_region, observations):
obs = observations[0]
bge = ReflectedRegionsBackgroundEstimator(on_region=bad_on_region, observations=obs)
bg = bge.process(obs)
return ObservationStats.from_observation(obs, bg)
from gammapy.background import ReflectedRegionsBackgroundEstimator
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
mask = data_store.obs_table["TARGET_NAME"] == "Crab"
obs_ids = data_store.obs_table["OBS_ID"][mask].data
observations = data_store.get_observations(obs_ids)
crab_position = SkyCoord(83.63, 22.01, unit="deg", frame="icrs")
# The ON region center is defined in the icrs frame. The angle is defined w.r.t. to its axis.
on_region = RectangleSkyRegion(center=crab_position,
width=0.5 * u.deg,
height=0.4 * u.deg,
angle=0 * u.deg)
background_estimator = ReflectedRegionsBackgroundEstimator(
observations=observations, on_region=on_region, min_distance=0.1 * u.rad)
background_estimator.run()
# Let's inspect the data extracted in the first observation
print(background_estimator.result[0])
background_estimator.plot()
# Now we change the ON region, and use a center defined in the galactic frame
on_region_galactic = RectangleSkyRegion(
center=crab_position.galactic,
width=0.5 * u.deg,
height=0.4 * u.deg,
angle=0 * u.deg,
)
# TODO: fit a Gaussian to the GC source
# ## Spectrum
#
# We'll run a spectral analysis using the classical reflected regions background estimation method,
# and using the on-off (often called WSTAT) likelihood function.
#
# ### Extraction
#
# The first step is to "extract" the spectrum, i.e. 1-dimensional counts and exposure and background vectors, as well as an energy dispersion matrix from the data and IRFs.
# In[24]:
bkg_estimator = ReflectedRegionsBackgroundEstimator(
obs_list=obs_list,
on_region=on_region,
exclusion_mask=exclusion_mask,
)
bkg_estimator.run()
bkg_estimate = bkg_estimator.result
bkg_estimator.plot()
# In[25]:
extract = SpectrumExtraction(
obs_list=obs_list,
bkg_estimate=bkg_estimate,
)
extract.run()
# ### Model fit
class SpectrumAnalysisIACT:
"""High-level analysis class to perform a full 1D IACT spectral analysis.
Observation selection must have happened before.
Config options:
* outdir : `pathlib.Path`, str
Output folder, None means no output
* background : dict
Forwarded to `~gammapy.background.ReflectedRegionsBackgroundEstimator`
* extraction : dict
Forwarded to `~gammapy.spectrum.SpectrumExtraction`
* fp_binning : `~astropy.units.Quantity`
Flux points binning
Parameters
----------
observations : `~gammapy.data.Observations`
Observations to analyse
config : dict
Config dict
"""
def __init__(self, observations, config):
self.observations = observations
self.config = config
def __str__(self):
ss = self.__class__.__name__
ss += "\n{}".format(self.observations)
ss += "\n{}".format(self.config)
return ss
def run(self, optimize_opts=None):
"""Run all steps."""
log.info("Running {}".format(self.__class__.__name__))
self.run_extraction()
self.run_fit(optimize_opts)
def run_extraction(self):
"""Run all steps for the spectrum extraction."""
self.background_estimator = ReflectedRegionsBackgroundEstimator(
observations=self.observations, **self.config["background"])
self.background_estimator.run()
self.extraction = SpectrumExtraction(
observations=self.observations,
bkg_estimate=self.background_estimator.result,
**self.config["extraction"])
self.extraction.run()
@property
def _result_dict(self):
"""Convert to dict."""
val = dict()
model = self.config["fit"]["model"]
val["model"] = model.to_dict()
fit_range = self.config["fit"].get("fit_range")
if fit_range is not None:
val["fit_range"] = dict(
min=fit_range[0].value,
max=fit_range[1].value,
unit=fit_range.unit.to_string("fits"),
)
val["statval"] = float(self.fit_result.total_stat)
val["statname"] = "wstat"
return val
def write(self, filename, mode="w"):
"""Write to YAML file.
Parameters
----------
filename : str
File to write
mode : str
Write mode
"""
d = self._result_dict
val = yaml.safe_dump(d, default_flow_style=False)
with open(str(filename), mode) as outfile:
outfile.write(val)
def run_fit(self, optimize_opts=None):
"""Run all step for the spectrum fit."""
fit_range = self.config["fit"].get("fit_range")
model = self.config["fit"]["model"]
for obs in self.extraction.spectrum_observations:
if fit_range is not None:
obs.mask_fit = obs.counts.energy_mask(fit_range[0],
fit_range[1])
obs.model = model
self.fit = Fit(self.extraction.spectrum_observations)
self.fit_result = self.fit.run(optimize_opts=optimize_opts)
model = self.config["fit"]["model"]
modelname = model.__class__.__name__
model.parameters.covariance = self.fit_result.parameters.covariance
filename = make_path(
self.config["outdir"]) / "fit_result_{}.yaml".format(modelname)
self.write(filename=filename)
obs_stacker = SpectrumDatasetOnOffStacker(
self.extraction.spectrum_observations)
obs_stacker.run()
datasets_fp = obs_stacker.stacked_obs
datasets_fp.model = model
self.flux_point_estimator = FluxPointsEstimator(
e_edges=self.config["fp_binning"], datasets=datasets_fp)
fp = self.flux_point_estimator.run()
fp.table["is_ul"] = fp.table["ts"] < 4
self.flux_points = fp
@property
def spectrum_result(self):
"""`~gammapy.spectrum.FluxPointsDataset`"""
return FluxPointsDataset(data=self.flux_points,
model=self.fit.datasets.datasets[0].model)
proj="TAN",
coordsys="GAL")
mask = exclusion_mask.geom.region_mask([exclusion_region], inside=False)
exclusion_mask.data = mask
exclusion_mask.plot()
# ## Estimate background
#
# Next we will manually perform a background estimate by placing [reflected regions](https://docs.gammapy.org/dev/background/reflected.html) around the pointing position and looking at the source statistics. This will result in a [gammapy.background.BackgroundEstimate](https://docs.gammapy.org/dev/api/gammapy.background.BackgroundEstimate.html) that serves as input for other classes in gammapy.
# In[ ]:
background_estimator = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_mask,
)
background_estimator.run()
# In[ ]:
plt.figure(figsize=(8, 8))
background_estimator.plot(add_legend=True)
# ## Source statistic
#
# Next we're going to look at the overall source statistics in our signal region. For more info about what debug plots you can create check out the [ObservationSummary](https://docs.gammapy.org/dev/api/gammapy.data.ObservationSummary.html#gammapy.data.ObservationSummary) class.
# In[ ]: