def test_validation_checks():
config = AnalysisConfig()
analysis = Analysis(config)
analysis.settings["observations"]["datastore"] = "other"
with pytest.raises(FileNotFoundError):
analysis.get_observations()
config = AnalysisConfig.from_template("1d")
analysis = Analysis(config)
assert analysis.get_flux_points() is False
assert analysis.run_fit() is False
assert analysis.set_model() is False
assert analysis.get_datasets() is False
analysis.get_observations()
analysis.settings["datasets"]["dataset-type"] = "not assigned"
assert analysis.get_datasets() is False
analysis.settings["datasets"]["dataset-type"] = "SpectrumDatasetOnOff"
analysis.get_observations()
analysis.get_datasets()
model_str = Path(MODEL_FILE).read_text()
analysis.set_model(model=model_str)
assert isinstance(analysis.model, SkyModels) is True
assert analysis.set_model() is False
analysis.run_fit()
del analysis.settings["flux-points"]
assert analysis.get_flux_points() is False
def test_validate_astropy_quantities():
config = AnalysisConfig()
cfg = {
"observations": {
"filters": [{
"filter_type": "all",
"lon": "1 deg"
}]
}
}
config.update_settings(cfg)
assert config.validate() is None
def test_config():
config = AnalysisConfig()
assert config.settings["general"]["logging"]["level"] == "INFO"
cfg = {"general": {"outdir": "test"}}
config.update_settings(cfg)
assert config.settings["general"]["logging"]["level"] == "INFO"
assert config.settings["general"]["outdir"] == "test"
with pytest.raises(ValueError):
Analysis()
assert "AnalysisConfig" in str(config)
def data_reduction(instrument):
log.info(f"data_reduction: {instrument}")
config = AnalysisConfig.read(f"config.yaml")
config.observations.datastore = f"$JOINT_CRAB/data/{instrument}"
config.datasets.stack = instrument_opts[instrument]['stack']
config.datasets.containment_correction = instrument_opts[instrument][
'containment']
config.datasets.on_region.radius = instrument_opts[instrument]['on_radius']
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
# TODO remove when safe mask can be set on config
if instrument is 'fact':
from gammapy.datasets import SpectrumDatasetOnOff
stacked = SpectrumDatasetOnOff.create(
e_reco=analysis.datasets[0]._energy_axis.edges,
e_true=analysis.datasets[0]._energy_axis.edges,
region=None)
for ds in analysis.datasets:
ds.mask_safe[:] = True
stacked.stack(ds)
analysis.datasets = Datasets([stacked])
analysis.datasets.write(f"reduced_{instrument}", overwrite=True)
def test_get_observations_obs_ids():
config = AnalysisConfig()
analysis = Analysis(config)
analysis.config.observations.datastore = "$GAMMAPY_DATA/cta-1dc/index/gps/"
analysis.config.observations.obs_ids = ["110380"]
analysis.get_observations()
assert len(analysis.observations) == 1
def test_analysis_3d_no_geom_irf():
config = AnalysisConfig.from_template("3d")
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
assert len(analysis.datasets) == 1
def test_analysis_3d_joint_datasets():
config = AnalysisConfig.from_template("3d")
config.settings["datasets"]["stack-datasets"] = False
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
assert len(analysis.datasets) == 4
def cli_run_analysis(filename, out, overwrite):
"""Performs automated data reduction process."""
config = AnalysisConfig.read(filename)
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
analysis.datasets.write(out, overwrite=overwrite)
log.info(f"Datasets stored in {out} folder.")
def test_get_observations_missing_irf():
config = AnalysisConfig()
analysis = Analysis(config)
analysis.config.observations.datastore = "$GAMMAPY_DATA/joint-crab/dl3/magic/"
analysis.config.observations.obs_ids = ["05029748"]
analysis.config.observations.required_irf = ["aeff", "edisp"]
analysis.get_observations()
assert len(analysis.observations) == 1
def main(config_path, models_path, output, reference):
config = AnalysisConfig.read(config_path)
analysis = Analysis(config)
log.info(config)
analysis.get_observations()
log.info(analysis)
log.info(dir(analysis))
log.info(analysis.datasets)
log.info(analysis.datasets[0].counts)
analysis.get_datasets()
analysis.read_models(models_path)
# stacked fit and flux estimation
analysis.run_fit()
analysis.get_flux_points()
# Plot flux points
ax_sed, ax_residuals = analysis.flux_points.plot_fit()
if reference:
plot_kwargs = {
"energy_range": [
analysis.config.flux_points.energy.min,
analysis.config.flux_points.energy.max,
],
"energy_power": 2,
"flux_unit": "erg-1 cm-2 s-1",
}
create_crab_spectral_model(reference).plot(
**plot_kwargs, ax=ax_sed, label="Crab reference"
)
ax_sed.legend()
ax_sed.set_ylim(1e-12, 1e-9)
base_out = Path(output)
ax_sed.get_figure().savefig(base_out.with_suffix(".pdf").as_posix())
plt.clf()
analysis.models.write(base_out.with_suffix(".yaml").as_posix(), overwrite=True)
analysis.flux_points.write(
base_out.with_suffix(".fits").as_posix(), overwrite=True
)
ax_excess = analysis.datasets["stacked"].plot_excess()
ax_excess.get_figure().savefig(base_out.with_suffix(".excess.pdf").as_posix())
plt.clf()
config.datasets.stack = False
analysis.get_observations()
analysis.get_datasets()
analysis.read_models(models_path)
lc_maker_low = LightCurveEstimator(
energy_edges=[.2, 5] * u.TeV, source=config.flux_points.source, reoptimize=False
)
lc_low = lc_maker_low.run(analysis.datasets)
ax_lc = lc_low.plot(marker="o", label="1D")
ax_lc.get_figure().savefig(base_out.with_suffix(".lc.pdf").as_posix())
plt.clf()
def test_update_config():
analysis = Analysis(AnalysisConfig())
data = {"general": {"outdir": "test"}}
config = AnalysisConfig(**data)
analysis.update_config(config)
assert analysis.config.general.outdir == "test"
analysis = Analysis(AnalysisConfig())
data = """
general:
outdir: test
"""
analysis.update_config(data)
assert analysis.config.general.outdir == "test"
analysis = Analysis(AnalysisConfig())
with pytest.raises(TypeError):
analysis.update_config(0)
def test_get_observations_obs_file(tmp_path):
config = AnalysisConfig()
analysis = Analysis(config)
analysis.get_observations()
filename = tmp_path / "obs_ids.txt"
filename.write_text("20136\n47829\n")
analysis.config.observations.obs_file = filename
analysis.get_observations()
assert len(analysis.observations) == 2
def target_config3d(config_file, target_config_file, tag):
"""Create analyis configuration for out source."""
targets_config_ = yaml.safe_load(open(target_config_file))
targets_config = {}
for conf in targets_config_: # define tag as key
targets_config[conf["tag"]] = conf
config = AnalysisConfig.from_yaml(config_file)
config_dict = config.settings
config_dict["observations"]["filters"][0]["frame"] = targets_config[tag][
"frame"]
config_dict["observations"]["filters"][0]["lon"] = targets_config[tag][
"lon"]
config_dict["observations"]["filters"][0]["lat"] = targets_config[tag][
"lat"]
config_dict["observations"]["filters"][0]["radius"] = targets_config[tag][
"radius"]
config_dict["observations"]["filters"][0]["border"] = targets_config[tag][
"radius"]
config_dict["datasets"]["geom"]["skydir"] = [
float(targets_config[tag]["lon"].strip(" deg")),
float(targets_config[tag]["lat"].strip(" deg")),
]
config_dict["datasets"]["geom"]["axes"][0]["lo_bnd"] = targets_config[tag][
"emin"]
config_dict["datasets"]["geom"]["axes"][0]["hi_bnd"] = targets_config[tag][
"emax"]
config_dict["datasets"]["geom"]["axes"][0]["nbin"] = targets_config[tag][
"nbin"]
config_dict["datasets"]["geom"]["axes"][0]["nbin"] = targets_config[tag][
"nbin"]
config_dict["flux-points"]["fp_binning"]["lo_bnd"] = targets_config[tag][
"emin"]
config_dict["flux-points"]["fp_binning"]["hi_bnd"] = targets_config[tag][
"emax"]
config_dict["flux-points"]["fp_binning"]["nbin"] = targets_config[tag][
"nbin"]
config_dict["flux-points"]["fp_binning"]["lo_bnd"] = targets_config[tag][
"emin"]
config_dict["flux-points"]["fp_binning"]["hi_bnd"] = targets_config[tag][
"emax"]
config_dict["flux-points"]["fp_binning"]["nbin"] = targets_config[tag][
"nbin"]
config_dict["fit"]["fit_range"]["min"] = str(
targets_config[tag]["emin"]) + " TeV"
config_dict["fit"]["fit_range"]["max"] = str(
targets_config[tag]["emax"]) + " TeV"
config.update_settings(config=config_dict)
return config
def run_analysis(estimate):
"""Run analysis from observation selection to model fitting."""
config = AnalysisConfig.read(f"{estimate}/config.yaml")
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
models = Models.read(f"{estimate}/models.yaml")
analysis.set_models(models)
analysis.run_fit()
return analysis
def test_get_observations_obs_time(tmp_path):
config = AnalysisConfig()
analysis = Analysis(config)
analysis.config.observations.obs_time = {
"start": "2004-03-26",
"stop": "2004-05-26",
}
analysis.get_observations()
assert len(analysis.observations) == 40
analysis.config.observations.obs_ids = [0]
with pytest.raises(ValueError):
analysis.get_observations()
def test_get_observations_obs_cone():
config = AnalysisConfig()
analysis = Analysis(config)
analysis.config.observations.datastore = "$GAMMAPY_DATA/hess-dl3-dr1"
analysis.config.observations.obs_cone = {
"frame": "icrs",
"lon": "83d",
"lat": "22d",
"radius": "5d",
}
analysis.get_observations()
assert len(analysis.observations) == 4
def setup_analysis(target):
log.info(f"analysis_3d_data_reduction: {target}")
opts = yaml.safe_load(open("targets.yaml"))[target]
txt = Path("config_template.yaml").read_text()
print(opts)
txt = txt.format_map(opts)
config = AnalysisConfig.from_yaml(txt)
config.flux_points.source = target
config.datasets.safe_mask.parameters = {"offset_max": 5 * u.deg}
return Analysis(config)
def test_analysis_1d_stacked():
config = AnalysisConfig.from_template("1d")
analysis = Analysis(config)
analysis.settings["datasets"]["stack-datasets"] = True
analysis.get_observations()
analysis.get_datasets()
analysis.set_model(filename=MODEL_FILE)
analysis.run_fit()
assert len(analysis.datasets) == 1
assert_allclose(analysis.datasets["stacked"].counts.data.sum(), 404)
pars = analysis.fit_result.parameters
assert_allclose(pars["index"].value, 2.689559, rtol=1e-3)
assert_allclose(pars["amplitude"].value, 2.81629e-11, rtol=1e-3)
def data_prep():
# source_pos = SkyCoord.from_name("MSH 15-52")
source_pos = SkyCoord(228.32, -59.08, unit="deg")
config = AnalysisConfig()
# Select observations - 2.5 degrees from the source position
config.observations.datastore = "$GAMMAPY_DATA/hess-dl3-dr1/"
config.observations.obs_cone = {
"frame": "icrs",
"lon": source_pos.ra,
"lat": source_pos.dec,
"radius": 2.5 * u.deg,
}
config.datasets.type = "3d"
config.datasets.geom.wcs.skydir = {
"lon": source_pos.ra,
"lat": source_pos.dec,
"frame": "icrs",
}
# The WCS geometry - centered on MSH 15-52
config.datasets.geom.wcs.width = {"width": "3 deg", "height": "3 deg"}
# The FoV radius to use for cutouts
config.datasets.geom.wcs.binsize = "0.02 deg"
config.datasets.geom.selection.offset_max = 3.5 * u.deg
# We now fix the energy axis for the counts map - (the reconstructed
# energy binning)
config.datasets.geom.axes.energy.min = "0.5 TeV"
config.datasets.geom.axes.energy.max = "5 TeV"
config.datasets.geom.axes.energy.nbins = 10
# We need to extract the ring for each observation separately, hence, no
# stacking at this stage
config.datasets.stack = False
# create the config
analysis = Analysis(config)
# for this specific case,w e do not need fine bins in true energy
analysis.config.datasets.geom.axes.energy_true = (
analysis.config.datasets.geom.axes.energy)
# `First get the required observations
analysis.get_observations()
# Analysis extraction
analysis.get_datasets()
return analysis
def test_analysis_1d(config_analysis_data):
config = AnalysisConfig.from_template("1d")
analysis = Analysis(config)
analysis.config.update_settings(config_analysis_data)
analysis.get_observations()
analysis.get_datasets()
analysis.set_model(filename=MODEL_FILE)
analysis.run_fit()
analysis.get_flux_points()
assert len(analysis.datasets) == 2
assert len(analysis.flux_points.data.table) == 4
dnde = analysis.flux_points.data.table["dnde"].quantity
assert dnde.unit == "cm-2 s-1 TeV-1"
assert_allclose(dnde[0].value, 8.03604e-12, rtol=1e-2)
assert_allclose(dnde[-1].value, 4.780021e-21, rtol=1e-2)
def analysis_3d_data_reduction(target):
log.info(f"analysis_3d_data_reduction: {target}")
opts = yaml.safe_load(open("targets.yaml"))[target]
txt = Path("config_template.yaml").read_text()
txt = txt.format_map(opts)
config = AnalysisConfig.from_yaml(txt)
analysis = Analysis(config)
analysis.get_observations()
log.info("Running data reduction")
analysis.get_datasets()
# TODO: write datasets and separate fitting to next function
# Not implemented in Gammapy yet, coming very soon.
log.info("Running fit ...")
analysis.read_models(f"{target}/model_3d.yaml")
logging.info(analysis.models)
analysis.run_fit()
logging.info(analysis.fit_result.parameters.to_table())
path = f"{target}/{target}_3d_bestfit.rst"
log.info(f"Writing {path}")
analysis.fit_result.parameters.to_table().write(path,
format="ascii.rst",
overwrite=True)
# analysis.get_flux_points(source=f"{target}")
# path = f"{target}/{target}_3d_fluxpoints.fits"
# log.info(f"Writing {path}")
# analysis.flux_points.write(path, overwrite=True)
analysis.get_flux_points(source=f"{target}")
path = f"{target}/{target}_3d_fluxpoints.ecsv"
log.info(f"Writing {path}")
keys = [
"e_ref", "e_min", "e_max", "dnde", "dnde_errp", "dnde_errn", "is_ul"
]
analysis.flux_points.data.table_formatted[keys].write(path,
format="ascii.ecsv",
overwrite=True)
return analysis # will write to disk when possible
def run(self):
raw = ""
section = self.arguments[0]
doc = AnalysisConfig._get_doc_sections()
for keyword in doc.keys():
if section == "" or section == keyword:
raw += doc[keyword]
include_lines = raw.splitlines()
codeblock = CodeBlock(
self.name,
[],
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine,
)
return codeblock.run()
def test_analysis_3d():
config = AnalysisConfig.from_template("3d")
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
analysis.set_model(filename=MODEL_FILE)
analysis.datasets["stacked"].background_model.tilt.frozen = False
analysis.run_fit()
analysis.get_flux_points()
assert len(analysis.datasets) == 1
assert len(analysis.fit_result.parameters) == 8
res = analysis.fit_result.parameters
assert res[3].unit == "cm-2 s-1 TeV-1"
assert len(analysis.flux_points.data.table) == 2
dnde = analysis.flux_points.data.table["dnde"].quantity
assert_allclose(dnde[0].value, 1.182768e-11, rtol=1e-1)
assert_allclose(dnde[-1].value, 4.051367e-13, rtol=1e-1)
assert_allclose(res["index"].value, 2.76607, rtol=1e-1)
assert_allclose(res["tilt"].value, -0.143204, rtol=1e-1)
def data_reduction(instrument):
log.info(f"data_reduction: {instrument}")
config = AnalysisConfig.read(f"config.yaml")
config.observations.datastore = str(Path().resolve().parent / "data" /
"joint-crab" / instrument)
config.datasets.stack = instrument_opts[instrument]["stack"]
config.datasets.containment_correction = instrument_opts[instrument][
"containment"]
config.datasets.on_region.radius = instrument_opts[instrument]["on_radius"]
if instrument == "fact":
config.datasets.safe_mask.methods = ["aeff-default"]
analysis = Analysis(config)
analysis.get_observations()
analysis.get_datasets()
if instrument == "fact":
counts = analysis.datasets[0].counts
data = counts.geom.energy_mask(emin=0.4 * u.TeV)
analysis.datasets[0].mask_safe = counts.copy(data=data)
analysis.datasets.write(f"reduced_{instrument}", overwrite=True)
def test_analysis_1d_stacked_no_fit_range():
cfg = """
observations:
datastore: $GAMMAPY_DATA/hess-dl3-dr1
obs_cone: {frame: icrs, lon: 83.633 deg, lat: 22.014 deg, radius: 5 deg}
obs_ids: [23592, 23559]
datasets:
type: 1d
stack: false
geom:
axes:
energy: {min: 0.01 TeV, max: 100 TeV, nbins: 73}
energy_true: {min: 0.03 TeV, max: 100 TeV, nbins: 50}
on_region: {frame: icrs, lon: 83.633 deg, lat: 22.014 deg, radius: 0.1 deg}
containment_correction: true
background:
method: reflected
"""
config = AnalysisConfig.from_yaml(cfg)
analysis = Analysis(config)
analysis.update_config(cfg)
analysis.config.datasets.stack = True
analysis.get_observations()
analysis.get_datasets()
analysis.read_models(MODEL_FILE_1D)
analysis.run_fit()
with pytest.raises(ValueError):
analysis.get_excess_map()
assert len(analysis.datasets) == 1
assert_allclose(analysis.datasets["stacked"].counts.data.sum(), 184)
pars = analysis.models.parameters
assert_allclose(analysis.datasets[0].mask_fit.data, True)
assert_allclose(pars["index"].value, 2.76913, rtol=1e-2)
assert_allclose(pars["amplitude"].value, 5.479729e-11, rtol=1e-2)
# In[ ]: from pathlib import Path from astropy import units as u from gammapy.analysis import Analysis, AnalysisConfig from gammapy.modeling.models import create_crab_spectral_model # ## Analysis configuration # # For configuration of the analysis we use the [YAML](https://en.wikipedia.org/wiki/YAML) data format. YAML is a machine readable serialisation format, that is also friendly for humans to read. In this tutorial we will write the configuration file just using Python strings, but of course the file can be created and modified with any text editor of your choice. # # Here is what the configuration for our analysis looks like: # In[ ]: config = AnalysisConfig() # the AnalysisConfig gives access to the various parameters used from logging to reduced dataset geometries print(config) # ### Setting the data to use # We want to use Crab runs from the H.E.S.S. DL3-DR1. We define here the datastore and a cone search of observations pointing with 5 degrees of the Crab nebula. # Parameters can be set directly or as a python dict. # In[ ]: # We define the datastore containing the data config.observations.datastore = "$GAMMAPY_DATA/hess-dl3-dr1" # We define the cone search parameters config.observations.obs_cone.frame = "icrs"
def test_get_observations_no_datastore():
config = AnalysisConfig()
analysis = Analysis(config)
analysis.config.observations.datastore = "other"
with pytest.raises(FileNotFoundError):
analysis.get_observations()
def get_example_config(which):
"""Example config: which can be 1d or 3d."""
return AnalysisConfig.read(CONFIG_PATH / f"example-{which}.yaml")
def run_analysis_3d(target_dict):
"""Run 3D analysis for the selected target"""
tag = target_dict["tag"]
name = target_dict["name"]
log.info(f"running 3d analysis, {tag}")
path_res = Path(tag + "/results/")
ra = target_dict["ra"]
dec = target_dict["dec"]
e_decorr = target_dict["e_decorr"]
config_str = f"""
general:
logging:
level: INFO
outdir: .
observations:
datastore: $GAMMAPY_DATA/hess-dl3-dr1/
filters:
- filter_type: par_value
value_param: {name}
variable: TARGET_NAME
datasets:
dataset-type: MapDataset
stack-datasets: true
offset-max: 2.5 deg
geom:
skydir: [{ra}, {dec}]
width: [5, 5]
binsz: 0.02
coordsys: CEL
proj: TAN
axes:
- name: energy
hi_bnd: 100
lo_bnd: 0.1
nbin: 24
interp: log
node_type: edges
unit: TeV
energy-axis-true:
name: energy
hi_bnd: 100
lo_bnd: 0.1
nbin: 72
interp: log
node_type: edges
unit: TeV
"""
print(config_str)
config = AnalysisConfig(config_str)
# Observation selection
analysis = Analysis(config)
analysis.get_observations()
if DEBUG is True:
analysis.observations.list = [analysis.observations.list[0]]
# Data reduction
analysis.get_datasets()
# Set runwise energy threshold. See reference paper, section 5.1.1.
for dataset in analysis.datasets:
# energy threshold given by the 10% edisp criterium
e_thr_bias = dataset.edisp.get_bias_energy(0.1)
# energy at which the background peaks
background_model = dataset.background_model
bkg_spectrum = background_model.map.get_spectrum()
peak = bkg_spectrum.data.max()
idx = list(bkg_spectrum.data).index(peak)
e_thr_bkg = bkg_spectrum.energy.center[idx]
esafe = max(e_thr_bias, e_thr_bkg)
dataset.mask_fit = dataset.counts.geom.energy_mask(emin=esafe)
# Model fitting
spatial_model = target_dict["spatial_model"]
model_config = f"""
components:
- name: {tag}
type: SkyModel
spatial:
type: {spatial_model}
frame: icrs
parameters:
- name: lon_0
value: {ra}
unit: deg
- name: lat_0
value: {dec}
unit: deg
spectral:
type: PowerLawSpectralModel
parameters:
- name: amplitude
value: 1.0e-12
unit: cm-2 s-1 TeV-1
- name: index
value: 2.0
unit: ''
- name: reference
value: {e_decorr}
unit: TeV
frozen: true
"""
model_npars = 5
if spatial_model == "DiskSpatialModel":
model_config = yaml.load(model_config)
parameters = model_config["components"][0]["spatial"]["parameters"]
parameters.append(
{
"name": "r_0",
"value": 0.2,
"unit": "deg",
"frozen": False
}
)
parameters.append(
{
"name": "e",
"value": 0.8,
"unit": "",
"frozen": False
}
)
parameters.append(
{
"name": "phi",
"value": 150,
"unit": "deg",
"frozen": False
}
)
parameters.append(
{
"name": "edge",
"value": 0.01,
"unit": "deg",
"frozen": True
}
)
model_npars += 4
analysis.set_model(model=model_config)
for dataset in analysis.datasets:
dataset.background_model.norm.frozen = False
analysis.run_fit()
parameters = analysis.model.parameters
parameters.covariance = analysis.fit_result.parameters.covariance[0:model_npars, 0:model_npars]
write_fit_summary(parameters, str(path_res / "results-summary-fit-3d.yaml"))
# Flux points
# TODO: This is a workaround to re-optimize the bkg in each energy bin. Add has to be added to the Analysis class
datasets = analysis.datasets.copy()
for dataset in datasets:
for par in dataset.parameters:
if par is not dataset.background_model.norm:
par.frozen = True
reoptimize = True if DEBUG is False else False
fpe = FluxPointsEstimator(
datasets=datasets, e_edges=FLUXP_EDGES, source=tag, reoptimize=reoptimize
)
flux_points = fpe.run()
flux_points.table["is_ul"] = flux_points.table["ts"] < 4
keys = ["e_ref", "e_min", "e_max", "dnde", "dnde_errp", "dnde_errn"]
flux_points.table_formatted[keys].write(
path_res / "flux-points-3d.ecsv", format="ascii.ecsv"
)
def cli_make_config(filename, overwrite):
"""Writes default configuration file."""
config = AnalysisConfig()
config.write(filename, overwrite=overwrite)
log.info(f"Configuration file produced: {filename}")
