def test_datastore_from_dir_no_obs_index(caplog, tmpdir):
"""Test the `from_dir` method."""
# Create small datastore and remove obs-index table
DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/").copy_obs([23523, 23592],
tmpdir)
os.remove(tmpdir / "obs-index.fits.gz")
data_store = DataStore.from_dir(tmpdir)
obs = data_store.obs(23523)
observations = data_store.get_observations()
assert data_store.obs_table is None
assert "No observation index table." in data_store.info(show=False)
assert obs.obs_info["ONTIME"] == 1687.0
assert len(observations) == 2
test_dir = tmpdir / "test"
os.mkdir(test_dir)
data_store.copy_obs([23523], test_dir)
data_store_copy = DataStore.from_dir(test_dir)
assert len(data_store_copy.obs_ids) == 1
assert data_store_copy.obs_table == None
def test_image_pipe(tmpdir):
tmpdir = str(tmpdir)
from subprocess import call
outdir = tmpdir
outdir2 = outdir + '/background'
cmd = 'mkdir -p {}'.format(outdir2)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2")
ds.copy_obs(ds.obs_table, tmpdir)
data_store = DataStore.from_dir(tmpdir)
bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2)
bgmaker.select_observations(selection='all')
bgmaker.group_observations()
bgmaker.make_model("2D")
bgmaker.save_models("2D")
fn = outdir2 + '/group-def.fits'
hdu_index_table = bgmaker.make_total_index_table(
data_store=data_store,
modeltype='2D',
out_dir_background_model=outdir2,
filename_obs_group_table=fn
)
fn = outdir + '/hdu-index.fits.gz'
hdu_index_table.write(fn, overwrite=True)
tmpdir = str(tmpdir)
center = SkyCoord(83.63, 22.01, unit='deg').galactic
energy_band = Energy([1, 10], 'TeV')
offset_band = Angle([0, 2.49], 'deg')
data_store = DataStore.from_dir(tmpdir)
# TODO: fix `binarize` implementation
# exclusion_mask = exclusion_mask.binarize()
image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg,
yref=center.b.deg, proj='TAN', coordsys='GAL')
refheader = image.to_image_hdu().header
exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits')
exclusion_mask = exclusion_mask.reproject(reference=refheader)
# Pb with the load psftable for one of the run that is not implemented yet...
data_store.hdu_table.remove_row(14)
mosaic = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store,
obs_table=data_store.obs_table, exclusion_mask=exclusion_mask)
mosaic.make_images(make_background_image=True, for_integral_flux=True, radius=10.)
assert_allclose(mosaic.maps['counts'].data.sum(), 2334.0, atol=3)
assert_allclose(mosaic.maps['bkg'].data.sum(), 1987.1513636663785, atol=3)
assert_allclose(mosaic.maps['exposure'].data.sum(), 54190569251987.68, atol=3)
assert_allclose(mosaic.maps['significance'].lookup(center), 33.707901541600634, atol=3)
assert_allclose(mosaic.maps['excess'].data.sum(), 346.8486363336217, atol=3)
def copy_data():
Path("data").mkdir(exist_ok=True)
ds_in = DataStore.from_dir(path_in)
ds_out = DataStore.from_dir(".")
for obs_id in obs_ids:
loc_in = ds_in.obs(obs_id).location(hdu_type="events")
loc_out = ds_out.obs(obs_id).location(hdu_type="events")
src = path_in / loc_in.file_dir / loc_in.file_name
dst = Path(loc_out.file_dir) / loc_out.file_name
print(f"cp {src} {dst}")
shutil.copy(src, dst)
def test_image_pipe(tmpdir):
tmpdir = str(tmpdir)
from subprocess import call
outdir = tmpdir
outdir2 = outdir + '/background'
cmd = 'mkdir -p {}'.format(outdir2)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2")
ds.copy_obs(ds.obs_table, tmpdir)
data_store = DataStore.from_dir(tmpdir)
bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2)
bgmaker.select_observations(selection='all')
bgmaker.group_observations()
bgmaker.make_model("2D")
bgmaker.save_models("2D")
fn = outdir2 + '/group-def.fits'
hdu_index_table = bgmaker.make_total_index_table(
data_store=data_store,
modeltype='2D',
out_dir_background_model=outdir2,
filename_obs_group_table=fn
)
fn = outdir + '/hdu-index.fits.gz'
hdu_index_table.write(fn, overwrite=True)
tmpdir = str(tmpdir)
center = SkyCoord(83.63, 22.01, unit='deg').galactic
energy_band = Energy([1, 10], 'TeV')
offset_band = Angle([0, 2.49], 'deg')
data_store = DataStore.from_dir(tmpdir)
# TODO: fix `binarize` implementation
# exclusion_mask = exclusion_mask.binarize()
image = SkyMap.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg,
yref=center.b.deg, proj='TAN', coordsys='GAL')
refheader = image.to_image_hdu().header
exclusion_mask = ExclusionMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits')
exclusion_mask = exclusion_mask.reproject(reference=refheader)
images = ImageAnalysis(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store,
obs_table=data_store.obs_table, exclusion_mask=exclusion_mask)
images.make_maps(radius=10., bkg_norm=True, spectral_index=2.3, for_integral_flux=True)
def test_datastore_from_dir():
"""Test the `from_dir` method."""
data_store_rel_path = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/",
"hdu-index.fits.gz",
"obs-index.fits.gz")
data_store_abs_path = DataStore.from_dir(
"$GAMMAPY_DATA/hess-dl3-dr1/",
"$GAMMAPY_DATA/hess-dl3-dr1/hdu-index.fits.gz",
"$GAMMAPY_DATA/hess-dl3-dr1/obs-index.fits.gz",
)
assert "Data store" in data_store_rel_path.info(show=False)
assert "Data store" in data_store_abs_path.info(show=False)
def cli_check_data2(dataset):
"""Check DL3 with DataStore.check
Write report in results/data_checks.
"""
from gammapy.data import DataStore
from joint_crab.utils import write_yaml
path = Path("results/data_checks")
path.mkdir(parents=True, exist_ok=True)
if dataset == "all":
names = ["magic", "hess", "fact", "veritas"]
else:
names = [dataset]
for name in names:
log.info(f"Running data checks for: {name}")
data_store = DataStore.from_dir(f"data/{name}")
results = data_store.check()
results = (_ for _ in results if _["level"] not in {"debug", "info"})
if name in ["magic", "fact", "veritas"]:
results = (_ for _ in results if _["msg"] != "Loading psf failed")
results = list(results)
filename = f"results/data_checks/check-data-{name}2.yaml"
write_yaml(results, filename)
def create_data(input_dir, dataset_config, exclusion_map=None):
telescope = dataset_config['telescope']
on_region_radius = dataset_config['on_radius']
energy_bins = dataset_config['e_reco_bins']
containment = dataset_config['containment_correction']
ds = DataStore.from_dir(os.path.join(input_dir, telescope))
observations = ds.get_observations(ds.obs_table['OBS_ID'].data)
t_obs = sum([o.observation_live_time_duration for o in observations])
# from IPython import embed; embed()
print(f'Total obstime for {telescope} is {t_obs.to("h")}')
source_position = dataset_config['source_position']
on_region = CircleSkyRegion(center=source_position,
radius=on_region_radius)
print('Estimating Background')
bkg_estimate = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_map)
bkg_estimate.run()
print('Extracting Count Spectra')
extract = SpectrumExtraction(
observations=observations,
bkg_estimate=bkg_estimate.result,
e_true=energy_bins,
e_reco=energy_bins,
containment_correction=containment,
use_recommended_erange=False, # TODO this might have to be checked.
)
extract.run()
return extract
def run_test_fitsexport(directory):
"""Run example analysis to test a fits data production
hap-data-fits-export crab has to be run in order to produce the example data
"""
log.info('Running test analysis of fits data')
from gammapy.data import DataStore
from gammapy.datasets import gammapy_extra
from gammapy.utils.scripts import read_yaml
from gammapy.spectrum.spectrum_pipe import run_spectrum_analysis_using_config
from gammapy.spectrum.results import SpectrumResult
s = DataStore.from_dir(directory)
print(s.info())
configfile = gammapy_extra.filename(
'test_datasets/spectrum/spectrum_analysis_example.yaml')
config = read_yaml(configfile)
config['extraction']['data']['datastore'] = directory
config['extraction']['data']['runlist'] = [23523, 23526, 23559, 23592]
fit, analysis = run_spectrum_analysis_using_config(config)
res = SpectrumResult(
fit=fit.result,
stats=analysis.observations.total_spectrum.spectrum_stats)
print(res.to_table())
def make_counts_image(dataset, max_runs):
log.info(f'Making all-sky image for dataset: {dataset}')
data_store = DataStore.from_dir(f'1dc/1dc/index/{dataset}')
image = SkyImage.empty(
nxpix=3600,
nypix=1800,
binsz=0.1,
xref=0,
yref=0,
proj='AIT',
coordsys='GAL',
)
# Define energy band
# energy_band = Energy([1, 10], 'TeV')
obs_ids = list(data_store.obs_table['OBS_ID'])
obs_ids = obs_ids[:max_runs] if max_runs > 0 else obs_ids
# Show a progress bar
from tqdm import tqdm
obs_ids = tqdm(obs_ids)
for obs_id in obs_ids:
events = data_store.obs(obs_id).events
image.fill_events(events)
image.data = image.data.astype('float32')
filename = f'checks/images/allsky_counts_{dataset}.fits.gz'
log.info(f'Writing {filename}')
image.write(filename, overwrite=True)
def make_image():
table = Table.read('acceptance_curve.fits')
table.pprint()
center = SkyCoord(83.63, 22.01, unit='deg').galactic
counts_image = make_empty_image(nxpix=1000, nypix=1000, binsz=0.01, xref=center.l.deg, yref=center.b.deg,
proj='TAN')
bkg_image = counts_image.copy()
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
for events in data_store.load_all("events"):
center = events.pointing_radec.galactic
livetime = events.observation_live_time_duration
solid_angle = Angle(0.01, "deg") ** 2
counts_image.data += bin_events_in_image(events, counts_image).data
#interp_param = dict(bounds_error=False, fill_value=None)
acc_hdu = fill_acceptance_image(bkg_image.header, center, table["offset"], table["Acceptance"])
acc = Quantity(acc_hdu.data, table["Acceptance"].unit) * solid_angle * livetime
bkg_image.data += acc.decompose()
print(acc.decompose().sum())
counts_image.writeto("counts_image.fits", clobber=True)
bkg_image.writeto("bkg_image.fits", clobber=True)
def make_image_from_2d_bg():
center = SkyCoord(83.63, 22.01, unit='deg').galactic
energy_band = Energy([1, 10], 'TeV')
offset_band = Angle([0, 2.49], 'deg')
data_store = DataStore.from_dir(
'/Users/jouvin/Desktop/these/temp/bg_model_image/')
# TODO: fix `binarize` implementation
# exclusion_mask = exclusion_mask.binarize()
image = SkyImage.empty(nxpix=250,
nypix=250,
binsz=0.02,
xref=center.l.deg,
yref=center.b.deg,
proj='TAN',
coordsys='GAL')
refheader = image.to_image_hdu().header
exclusion_mask = SkyMask.read(
'$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits')
exclusion_mask = exclusion_mask.reproject(reference=refheader)
mosaic_images = StackedObsImageMaker(image,
energy_band=energy_band,
offset_band=offset_band,
data_store=data_store,
obs_table=data_store.obs_table,
exclusion_mask=exclusion_mask)
mosaic_images.make_images(make_background_image=True,
for_integral_flux=True,
radius=10.,
make_psf=True,
region_center=center)
filename = 'fov_bg_images.fits'
log.info('Writing {}'.format(filename))
mosaic_images.images.write(filename, clobber=True)
def make_bg_model_two_groups():
from subprocess import call
outdir = '/Users/jouvin/Desktop/these/temp/bg_model_image/'
outdir2 = outdir + '/background'
cmd = 'mkdir -p {}'.format(outdir2)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
cmd = 'cp -r $GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/ {}'.format(outdir)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/')
bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2)
bgmaker.select_observations(selection='all')
bgmaker.group_observations()
bgmaker.make_model("2D")
bgmaker.save_models("2D")
fn = outdir2 + '/group-def.fits'
hdu_index_table = bgmaker.make_total_index_table(
data_store=data_store,
modeltype='2D',
out_dir_background_model=outdir2,
filename_obs_group_table=fn
)
fn = outdir + '/hdu-index.fits.gz'
hdu_index_table.write(fn, overwrite=True)
def run_benchmark():
info = {"n_obs": N_OBS}
t = time.time()
data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/")
OBS_ID = 110380
obs_ids = OBS_ID * np.ones(N_OBS)
observations = data_store.get_observations(obs_ids)
info["data_loading"] = time.time() - t
t = time.time()
m = Map.create()
for obs in observations:
m.fill_events(obs.events)
info["filling"] = time.time() - t
t = time.time()
m.write("survey_map.fits.gz", overwrite=True)
info["writing"] = time.time() - t
with open("bench.yaml", "w") as fh:
yaml.dump(info, fh, sort_keys=False, indent=4)
def make_bg_model_two_groups():
from subprocess import call
outdir = '/Users/jouvin/Desktop/these/temp/bg_model_image/'
outdir2 = outdir + '/background'
cmd = 'mkdir -p {}'.format(outdir2)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
cmd = 'cp -r $GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/ {}'.format(
outdir)
print('Executing: {}'.format(cmd))
call(cmd, shell=True)
data_store = DataStore.from_dir(
'$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/')
bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2)
bgmaker.select_observations(selection='all')
bgmaker.group_observations()
bgmaker.make_model("2D")
bgmaker.save_models("2D")
fn = outdir2 + '/group-def.fits'
hdu_index_table = bgmaker.make_total_index_table(
data_store=data_store,
modeltype='2D',
out_dir_background_model=outdir2,
filename_obs_group_table=fn)
fn = outdir + '/hdu-index.fits.gz'
hdu_index_table.write(fn, overwrite=True)
def load_data(input_dir, dataset_config, exclusion_map=None):
on_region_radius = dataset_config['on_radius']
e_reco_bins = dataset_config['e_reco_bins']
e_true_bins = dataset_config['e_true_bins']
containment = dataset_config['containment_correction']
ds = DataStore.from_dir(input_dir)
observations = ds.get_observations(ds.obs_table['OBS_ID'].data)
# observations = ds.get_observations(ds.hdu_table['OBS_ID'].data) # this is plenty wrong
source_position = dataset_config['source_position']
on_region = CircleSkyRegion(center=source_position,
radius=on_region_radius)
print('Estimating Background')
bkg_estimate = ReflectedRegionsBackgroundEstimator(
observations=observations,
on_region=on_region,
exclusion_mask=exclusion_map)
bkg_estimate.run()
print('Extracting Count Spectra')
extract = SpectrumExtraction(
observations=observations,
bkg_estimate=bkg_estimate.result,
e_true=e_true_bins,
e_reco=e_reco_bins,
containment_correction=containment,
use_recommended_erange=False,
)
extract.run()
if dataset_config['stack']:
return [extract.spectrum_observations.stack()]
else:
return extract.spectrum_observations
def test_datastore_fixed_rad_max():
data_store = DataStore.from_dir("$GAMMAPY_DATA/joint-crab/dl3/magic")
observations = data_store.get_observations([5029748],
required_irf=['aeff', 'edisp'])
assert len(observations) == 1
obs = observations[0]
assert obs.rad_max is not None
assert obs.rad_max.quantity.shape == (1, 1)
assert u.allclose(obs.rad_max.quantity, np.sqrt(0.02) * u.deg)
# test it also works with edisp (removing aeff)
obs = data_store.get_observations([5029748],
required_irf=['aeff', 'edisp'])[0]
obs.aeff = None
assert obs.rad_max is not None
assert obs.rad_max.quantity.shape == (1, 1)
assert u.allclose(obs.rad_max.quantity, 0.1414213 * u.deg)
# removing the last irf means we have no rad_max info
obs = data_store.get_observations([5029748],
required_irf=['aeff', 'edisp'])[0]
obs.aeff = None
obs.edisp = None
assert obs.rad_max is None
def make_new_directorydataset_listobs(nobs, config_directory, source_name,
center, obsdir, list_obs):
"""
Creates a directory with only the run used for the Images of the source and create a new index table with the
background aceeptance curve location to used for the bkg image.
Used a list of observation not a selection at less tat 2deg from the pointing position
Parameters
----------
nobs: number of observation you want
config_directory: name of the config chains used to produce the data
source_name: name of the source you want to compute the image
center: SkyCoord of the source
obsdir: directory where you want to put these data
list_obs: list of obs id we want to create the new data_store
Returns
-------
"""
ds = DataStore.from_dir(config_directory)
list_ind = list()
for obs in list_obs:
try:
i = np.where(ds.obs_table["OBS_ID"] == obs)[0][0]
list_ind.append(i)
except Exception:
print(obs)
continue
try:
shutil.rmtree(obsdir)
except Exception:
pass
ds.copy_obs(ds.obs_table[list_ind], obsdir)
def make_image():
table = Table.read('acceptance_curve.fits')
table.pprint()
center = SkyCoord(83.63, 22.01, unit='deg').galactic
counts_image = SkyMap.empty(nxpix=1000,
nypix=1000,
binsz=0.01,
xref=center.l.deg,
yref=center.b.deg,
proj='TAN').to_image_hdu()
bkg_image = counts_image.copy()
data_store = DataStore.from_dir(
'$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
for events in data_store.load_all("events"):
center = events.pointing_radec.galactic
livetime = events.observation_live_time_duration
solid_angle = Angle(0.01, "deg")**2
counts_image.data += bin_events_in_image(events, counts_image).data
# interp_param = dict(bounds_error=False, fill_value=None)
acc_hdu = fill_acceptance_image(bkg_image.header, center,
table["offset"], table["Acceptance"])
acc = Quantity(acc_hdu.data,
table["Acceptance"].unit) * solid_angle * livetime
bkg_image.data += acc.decompose()
print(acc.decompose().sum())
counts_image.writeto("counts_image_save.fits", clobber=True)
bkg_image.writeto("bkg_image_save.fits", clobber=True)
def __init__(self, prod):
self.ref_dict = prod
self.ds = DataStore.from_dir(prod["datastore"])
self.ref_energy = 1 * u.TeV
self.ref_offset = 0.25 * u.deg
self.ref_rad = np.arange(0, 2, 0.1) * u.deg
self.ref_migra = 0.95
self.observation = self.ds.obs(prod["test_obs"])
def get_DataStore(self):
"""get gammapy DataStore object"""
if self.name in ["fermi", "joint"]:
return None
elif self.name in ["magic", "hess", "fact", "veritas"]:
return DataStore.from_dir(self.data_path)
else:
raise ValueError(f"No datastore for: {self.name}")
def data_prep():
data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/")
OBS_ID = 110380
obs_ids = OBS_ID * np.ones(N_OBS)
observations = data_store.get_observations(obs_ids)
energy_axis = MapAxis.from_bounds(0.1,
10,
nbin=10,
unit="TeV",
name="energy",
interp="log")
geom = WcsGeom.create(
skydir=(0, 0),
binsz=0.02,
width=(10, 8),
frame="galactic",
proj="CAR",
axes=[energy_axis],
)
energy_axis_true = MapAxis.from_bounds(0.05,
20,
nbin=30,
unit="TeV",
name="energy_true",
interp="log")
offset_max = 4 * u.deg
maker = MapDatasetMaker()
safe_mask_maker = SafeMaskMaker(methods=["offset-max"],
offset_max=offset_max)
stacked = MapDataset.create(geom=geom, energy_axis_true=energy_axis_true)
spatial_model = PointSpatialModel(lon_0="-0.05 deg",
lat_0="-0.05 deg",
frame="galactic")
spectral_model = ExpCutoffPowerLawSpectralModel(
index=2,
amplitude=3e-12 * u.Unit("cm-2 s-1 TeV-1"),
reference=1.0 * u.TeV,
lambda_=0.1 / u.TeV,
)
model = SkyModel(spatial_model=spatial_model,
spectral_model=spectral_model,
name="gc-source")
datasets = Datasets([])
for idx, obs in enumerate(observations):
cutout = stacked.cutout(obs.pointing_radec,
width=2 * offset_max,
name=f"dataset{idx}")
dataset = maker.run(cutout, obs)
dataset = safe_mask_maker.run(dataset, obs)
dataset.models = [model, FoVBackgroundModel(dataset_name=dataset.name)]
datasets.append(dataset)
return datasets
def data_prep():
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
OBS_ID = 23523
obs_ids = OBS_ID * np.ones(N_OBS)
observations = data_store.get_observations(obs_ids)
target_position = SkyCoord(ra=83.63, dec=22.01, unit="deg", frame="icrs")
on_region_radius = Angle("0.11 deg")
on_region = CircleSkyRegion(center=target_position, radius=on_region_radius)
exclusion_region = CircleSkyRegion(
center=SkyCoord(183.604, -8.708, unit="deg", frame="galactic"),
radius=0.5 * u.deg,
)
skydir = target_position.galactic
exclusion_mask = Map.create(
npix=(150, 150), binsz=0.05, skydir=skydir, proj="TAN", coordsys="GAL"
)
mask = exclusion_mask.geom.region_mask([exclusion_region], inside=False)
exclusion_mask.data = mask
e_reco = MapAxis.from_bounds(0.1, 40, nbin=40, interp="log", unit="TeV").edges
e_true = MapAxis.from_bounds(0.05, 100, nbin=200, interp="log", unit="TeV").edges
dataset_maker = SpectrumDatasetMaker(
region=on_region, e_reco=e_reco, e_true=e_true, containment_correction=True
)
bkg_maker = ReflectedRegionsBackgroundMaker(exclusion_mask=exclusion_mask)
safe_mask_masker = SafeMaskMaker(methods=["aeff-max"], aeff_percent=10)
spectral_model = PowerLawSpectralModel(
index=2, amplitude=2e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.TeV
)
spatial_model = PointSpatialModel(
lon_0=target_position.ra, lat_0=target_position.dec, frame="icrs"
)
spatial_model.lon_0.frozen = True
spatial_model.lat_0.frozen = True
sky_model = SkyModel(
spatial_model=spatial_model, spectral_model=spectral_model, name=""
)
# Data preparation
datasets = []
for ind, observation in enumerate(observations):
dataset = dataset_maker.run(observation, selection=["counts", "aeff", "edisp"])
dataset_on_off = bkg_maker.run(dataset, observation)
dataset_on_off = safe_mask_masker.run(dataset_on_off, observation)
dataset_on_off.name = f"dataset{ind}"
dataset_on_off.models = sky_model
datasets.append(dataset_on_off)
return Datasets(datasets)
def get_observations():
data_store = DataStore.from_dir("$CTADATA/index/gc")
t = data_store.obs_table
pos_obs = SkyCoord(t["RA_PNT"], t["DEC_PNT"], unit="deg")
offset = config.target_skycoord.separation(pos_obs)
mask = offset < config.offset_max
obs_id = t["OBS_ID"][mask]
if config.debug:
obs_id = obs_id[:3]
return data_store.obs_list(obs_id)
def get_observations(self):
"""Fetch observations from the data store according to criteria defined in the configuration."""
observations_settings = self.config.observations
path = make_path(observations_settings.datastore)
if path.is_file():
self.datastore = DataStore.from_file(path)
elif path.is_dir():
self.datastore = DataStore.from_dir(path)
else:
raise FileNotFoundError(f"Datastore not found: {path}")
log.info("Fetching observations.")
if (len(observations_settings.obs_ids)
and observations_settings.obs_file is not None):
raise ValueError(
"Values for both parameters obs_ids and obs_file are not accepted."
)
elif (not len(observations_settings.obs_ids)
and observations_settings.obs_file is None):
obs_list = self.datastore.get_observations()
ids = [obs.obs_id for obs in obs_list]
elif len(observations_settings.obs_ids):
obs_list = self.datastore.get_observations(
observations_settings.obs_ids)
ids = [obs.obs_id for obs in obs_list]
else:
path = make_path(observations_settings.obs_file)
ids = list(
Table.read(path, format="ascii", data_start=0).columns[0])
if observations_settings.obs_cone.lon is not None:
cone = dict(
type="sky_circle",
frame=observations_settings.obs_cone.frame,
lon=observations_settings.obs_cone.lon,
lat=observations_settings.obs_cone.lat,
radius=observations_settings.obs_cone.radius,
border="0 deg",
)
selected_cone = self.datastore.obs_table.select_observations(cone)
ids = list(set(ids) & set(selected_cone["OBS_ID"].tolist()))
self.observations = self.datastore.get_observations(ids,
skip_missing=True)
if observations_settings.obs_time.start is not None:
start = observations_settings.obs_time.start
stop = observations_settings.obs_time.stop
if len(start.shape) == 0:
time_intervals = [(start, stop)]
else:
time_intervals = [(tstart, tstop)
for tstart, tstop in zip(start, stop)]
self.observations = self.observations.select_time(time_intervals)
log.info(f"Number of selected observations: {len(self.observations)}")
for obs in self.observations:
log.debug(obs)
def make_summary_latex():
"""Make summary LaTeX table for the paper."""
path = "results/summary/data.tex"
log.info(f"Writing {path}")
table = open(path, "w")
table.write(r"\begin{tabular}{lrrrrrl}")
table.write("\n")
table.write(r"\hline")
table.write("\n")
table.write(
r"Dataset & $T_{\rm obs}$ & $E_{\rm min}$ & $E_{\rm max}$ & $N_{\rm on}$ & $N_{\rm bkg}$ & $R_{\rm on}$ \\"
)
table.write("\n")
table.write(
r" & & TeV & TeV & & & deg \\ \hline"
)
table.write("\n")
for name in config.all_datasets:
dataset = config.datasets[name]
e_min_list = []
for _spec in dataset.get_SpectrumObservationList():
e_min_list.append(_spec.lo_threshold)
spec = dataset.get_SpectrumObservationList().stack()
stats = spec.total_stats
row_name = rf"\{name} & "
if name == "fermi":
T_obs = r"$\sim$7 yr & "
e_min = dataset.energy_range[0].to("TeV").value
E_min = f"{e_min:.2f} & "
else:
data_store = DataStore.from_dir(f"data/{dataset.name}")
ontime = sum(data_store.obs_table["ONTIME"]) * u.s
ontime = ontime.to("h").value
T_obs = f"{ontime:.2f} h & "
# in case of the IACT e_min is taken from the staked obs
e_min = min(e_min_list)
E_min = f"{e_min.to('TeV').value:.2f} & "
e_max = dataset.energy_range[1].to("TeV").value
E_max = f"{e_max:.0f} & "
N_on = f"{stats.n_on} & "
N_bkg = f"{stats.background:.1f} & "
r_on = dataset.on_radius.to("deg").value
R_on = rf"{r_on:.2f} \\"
table.write(row_name + T_obs + E_min + E_max + N_on + N_bkg + R_on)
table.write("\n")
table.write("\hline")
table.write("\n")
table.write("\end{tabular}")
table.close()
def make_counts_array():
"""Make an example counts array with energy and offset axes."""
data_store = DataStore.from_dir(gammapy_extra.dir / 'datasets/hess-crab4')
event_lists = data_store.load_all('events')
ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV')
offset = np.linspace(0, 2.5, 100)
array = EnergyOffsetArray(ebounds, offset)
array.fill_events(event_lists)
return array
def main():
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1")
obs_id = data_store.obs_table["OBS_ID"]
observations = data_store.get_observations(obs_id)
m = Map.create()
for obs in observations:
log.info(f"Processing obs_id: {obs.obs_id}")
m.fill_events(obs.events)
m.write("survey_map.fits.gz")
def make_counts_array():
"""Make an example counts array with energy and offset axes."""
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
event_lists = data_store.load_all('events')
ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV')
offset = Angle(np.linspace(0, 2.5, 100), "deg")
array = EnergyOffsetArray(ebounds, offset)
array.fill_events(event_lists)
return array
def get_irf(self):
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1")
obs = data_store.obs(47828)
# Using IRF for 30 deg zenith angle - can be changed later
irfs = {
'aeff': obs.aeff,
'edisp': obs.edisp,
'psf': obs.psf,
'bkg': obs.bkg
}
return irfs
def _set_data_store(self):
"""Set the datastore on the Analysis object."""
path = make_path(self.config.observations.datastore)
if path.is_file():
log.debug(f"Setting datastore from file: {path}")
self.datastore = DataStore.from_file(path)
elif path.is_dir():
log.debug(f"Setting datastore from directory: {path}")
self.datastore = DataStore.from_dir(path)
else:
raise FileNotFoundError(f"Datastore not found: {path}")
def data_prep():
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
OBS_ID = 23523
obs_ids = OBS_ID * np.ones(N_OBS)
observations = data_store.get_observations(obs_ids)
target_position = SkyCoord(ra=83.63308, dec=22.01450, unit="deg")
e_reco = MapAxis.from_bounds(0.1, 40, nbin=40, interp="log",
unit="TeV").edges
e_true = MapAxis.from_bounds(0.05, 100, nbin=200, interp="log",
unit="TeV").edges
on_region_radius = Angle("0.11 deg")
on_region = CircleSkyRegion(center=target_position,
radius=on_region_radius)
dataset_maker = SpectrumDatasetMaker(containment_correction=True,
selection=["counts", "aeff", "edisp"])
empty = SpectrumDatasetOnOff.create(region=on_region,
e_reco=e_reco,
e_true=e_true)
bkg_maker = ReflectedRegionsBackgroundMaker()
safe_mask_masker = SafeMaskMaker(methods=["aeff-max"], aeff_percent=10)
spectral_model = PowerLawSpectralModel(index=2.6,
amplitude=2.0e-11 *
u.Unit("1 / (cm2 s TeV)"),
reference=1 * u.TeV)
spectral_model.index.frozen = False
model = spectral_model.copy()
model.name = "crab"
datasets_1d = []
for observation in observations:
dataset = dataset_maker.run(dataset=empty.copy(),
observation=observation)
dataset_on_off = bkg_maker.run(dataset, observation)
dataset_on_off = safe_mask_masker.run(dataset_on_off, observation)
datasets_1d.append(dataset_on_off)
for dataset in datasets_1d:
model = spectral_model.copy()
model.name = "crab"
dataset.model = model
return datasets_1d
def test_datastore_copy_obs(tmp_path, data_store):
data_store.copy_obs([23523, 23592], tmp_path, overwrite=True)
substore = DataStore.from_dir(tmp_path)
assert str(substore.hdu_table.base_dir) == str(tmp_path)
assert len(substore.obs_table) == 2
desired = data_store.obs(23523)
actual = substore.obs(23523)
assert str(actual.events.table) == str(desired.events.table)
def test_fill_cube():
filename = '$GAMMAPY_EXTRA/test_datasets/background/bg_cube_model_test1.fits'
array = FOVCube.read(filename, format='table', scheme='bg_cube')
array.data = Quantity(np.zeros_like(array.data.value), 'u')
print(type(array.data))
dir = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2'
data_store = DataStore.from_dir(dir)
ev_list = data_store.load_all('events')
array.fill_events(ev_list)
array.write('test_background.fits', format='image', clobber=True)
def test_obs():
# data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-pa')
data_store.info()
obs = data_store.obs(obs_id=23523)
# import IPython; IPython.embed()
# obs.info()
# print(type(obs.events))
# print(type(obs.gti))
print(type(obs.aeff))
print(type(obs.edisp))
print(type(obs.psf))
def test_obs():
# data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-pa')
data_store.info()
obs = data_store.obs(obs_id=23523)
# import IPython; IPython.embed()
# obs.info()
# print(type(obs.events))
# print(type(obs.gti))
print(type(obs.aeff))
print(type(obs.edisp))
print(type(obs.psf))
def select_data(target_position):
data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
selection = dict(
type="sky_circle",
frame="icrs",
lon=target_position.ra,
lat=target_position.dec,
radius=2 * u.deg,
)
obs_ids = data_store.obs_table.select_observations(selection)["OBS_ID"]
observations = data_store.get_observations(obs_ids)
return observations
def make_model():
dir = str(gammapy_extra.dir) + '/datasets/hess-crab4-hd-hap-prod2'
data_store = DataStore.from_dir(dir)
obs_table = data_store.obs_table
ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV')
offset = sqrt_space(start=0, stop=2.5, num=100) * u.deg
excluded_sources = make_excluded_sources()
multi_array = EnergyOffsetBackgroundModel(ebounds, offset)
multi_array.fill_obs(obs_table, data_store, excluded_sources)
#multi_array.fill_obs(obs_table, data_store)
multi_array.compute_rate()
bgarray = multi_array.bg_rate
energy_range = Energy([1, 10], 'TeV')
table = bgarray.acceptance_curve_in_energy_band(energy_range, energy_bins=10)
multi_array.write('energy_offset_array.fits', overwrite=True)
table.write('acceptance_curve.fits', overwrite=True)
def make_image_from_2d_bg():
center = SkyCoord(83.63, 22.01, unit='deg').galactic
energy_band = Energy([1, 10], 'TeV')
offset_band = Angle([0, 2.49], 'deg')
data_store = DataStore.from_dir('/Users/jouvin/Desktop/these/temp/bg_model_image/')
# TODO: fix `binarize` implementation
# exclusion_mask = exclusion_mask.binarize()
image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg,
yref=center.b.deg, proj='TAN', coordsys='GAL')
refheader = image.to_image_hdu().header
exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits')
exclusion_mask = exclusion_mask.reproject(reference=refheader)
mosaic_images = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store,
obs_table=data_store.obs_table, exclusion_mask=exclusion_mask)
mosaic_images.make_images(make_background_image=True, for_integral_flux=True, radius=10., make_psf = True,
region_center=center)
filename = 'fov_bg_maps.fits'
log.info('Writing {}'.format(filename))
mosaic_images.maps.write(filename, clobber=True)
def run_test_fitsexport(directory):
"""Run example analysis to test a fits data production
hap-data-fits-export crab has to be run in order to produce the example data
"""
log.info('Running test analysis of fits data')
from gammapy.data import DataStore
from gammapy.datasets import gammapy_extra
from gammapy.utils.scripts import read_yaml
from gammapy.spectrum.spectrum_pipe import run_spectrum_analysis_using_config
from gammapy.spectrum.results import SpectrumResult
s = DataStore.from_dir(directory)
print(s.info())
configfile = gammapy_extra.filename(
'test_datasets/spectrum/spectrum_analysis_example.yaml')
config = read_yaml(configfile)
config['extraction']['data']['datastore'] = directory
config['extraction']['data']['runlist'] = [23523, 23526, 23559, 23592]
fit, analysis = run_spectrum_analysis_using_config(config)
res = SpectrumResult(fit=fit.result, stats=analysis.observations.total_spectrum.spectrum_stats)
print(res.to_table())
def run_benchmark():
# Set up data store and select N_OBS times the observation OBS_ID
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/test_datasets/cta_1dc/')
obs_ids = OBS_ID * np.ones(N_OBS)
obs_list = data_store.obs_list(obs_id=obs_ids)
target_position = SkyCoord(0, 0, unit='deg', frame='galactic')
on_radius = 0.2 * u.deg
on_region = CircleSkyRegion(center=target_position, radius=on_radius)
bkg_estimator = RingBackgroundEstimator(
r_in=0.5 * u.deg,
width=0.2 * u.deg,
)
# Define reference image centered on the target
xref = target_position.galactic.l.value
yref = target_position.galactic.b.value
ref_image = SkyImage.empty(
nxpix=800, nypix=600, binsz=0.02,
xref=xref, yref=yref,
proj='TAN', coordsys='GAL',
)
exclusion_mask = ref_image.region_mask(on_region)
exclusion_mask.data = 1 - exclusion_mask.data
image_estimator = IACTBasicImageEstimator(
reference=ref_image,
emin=100 * u.GeV,
emax=100 * u.TeV,
offset_max=3 * u.deg,
background_estimator=bkg_estimator,
exclusion_mask=exclusion_mask,
)
result = image_estimator.run(obs_list)
def make_reco_model():
"""Make a reco bg cube model."""
METHOD = 'default'
data_dir = 'test_dataset'
overwrite = OVERWRITE
test = TEST
group_id = GROUP_ID
# create output folder
outdir = os.path.join(OUTDIR, 'reco')
_create_dir(outdir, overwrite)
# 0. create dummy observation grouping
obs_groups = create_dummy_observation_grouping()
# save
outfile = os.path.join(outdir, 'bg_observation_groups.ecsv')
print('Writing {}'.format(outfile, overwrite=overwrite))
obs_groups.write(outfile)
# 1. create dummy dataset
# use enough stats so that rebinning (and resmoothing?) doesn't take place
n_obs = 100
if test:
# run fast
n_obs = 2
az_range = AZ_RANGE
alt_range = ALT_RANGE
random_state = np.random.RandomState(seed=0)
sigma = SIGMA
spectral_index = INDEX
make_test_dataset(outdir=data_dir, overwrite=overwrite,
observatory_name='HESS', n_obs=n_obs,
az_range=az_range,
alt_range=alt_range,
date_range=(Time('2010-01-01T00:00:00',
format='isot', scale='utc'),
Time('2015-01-01T00:00:00',
format='isot', scale='utc')),
n_tels_range=(3, 4),
sigma=sigma,
spectral_index=spectral_index,
random_state=random_state)
# 2. get observation table
data_store = DataStore.from_dir(dir=data_dir)
observation_table = data_store.make_observation_table()
outfile = os.path.join(outdir, 'bg_observation_table_group{}.fits.gz'.format(group_id))
print("Writing {}".format(outfile))
observation_table.write(outfile, overwrite=overwrite)
# 3. build bg model
method = METHOD
bg_cube_model = make_bg_cube_model(observation_table=observation_table,
data_dir=data_dir,
method=method,
do_not_force_mev_units=True)
# save
outfile = os.path.join(outdir, 'bg_cube_model_group{}'.format(group_id))
print("Writing {}".format('{}_table.fits.gz'.format(outfile)))
print("Writing {}".format('{}_image.fits.gz'.format(outfile)))
bg_cube_model.write('{}_table.fits.gz'.format(outfile),
format='table', clobber=overwrite)
bg_cube_model.write('{}_image.fits.gz'.format(outfile),
format='image', clobber=overwrite)
def proceeding_plot(obs):
# import IPython; IPython.embed()
fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(12, 4))
obs.aeff.plot_energy_dependence(ax=axes[0])
edisp = obs.edisp.to_energy_dispersion(offset='1 deg')
# edisp.plot_matrix(ax=axes[1])
psf_edep = obs.psf.to_table_psf(theta='1 deg')
for energy in [1, 3] * u.TeV:
psf = psf_edep.table_psf_at_energy(energy)
psf.plot_psf_vs_theta()
plt.xscale('linear')
plt.yscale('linear')
plt.xlim(0, 0.5)
fig.tight_layout()
filename = 'iact-dl3.pdf'
print('Writing ', filename)
fig.savefig(filename)
if __name__ == '__main__':
data_store = DataStore.from_dir('$HOME/code/HESS-DL3-DR1/release_store/')
obs = data_store.obs(obs_id=23592)
proceeding_plot(obs)
peek_plots(obs)
"""Add phase for two obs on Vela pulsar from CTA DC-1"""
import numpy as np
import astropy.units as u
from astropy.time import Time
from astropy.coordinates import SkyCoord
from gammapy.data import DataStore
###########################################################################
####################### Extracting Vela files ###########################
###########################################################################
# Load the data store (contains the information about all the DC1 data)
# Assumes you have an environment variable CTADATA set pointing to the DC1 folder
data_store = DataStore.from_dir('$CTADATA/index/gps')
table = data_store.obs_table
# Defining the offset as the angular separation between the observation position and the target position
pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg')
pos_target = SkyCoord.from_name('vela')
offset = pos_target.separation(pos_obs)
# Defining a mask to select all runs targeting Vela with an offset < 2 deg
mask = (offset < 2 * u.deg)
# Applying the mask
table = table[mask]
# Getting the list of observation indices of Vela
obs_ids = table['OBS_ID']
"""Example how to load and take a peek at an IACT observation.
"""
import matplotlib.pyplot as plt
from gammapy.data import DataStore
from gammapy.utils.mpl_style import gammapy_mpl_style
# TODO: Update once this issue is resolved:
# https://github.com/astropy/astropy/issues/4140
# plt.style.use(gammapy_mpl_style)
plt.rcParams.update(gammapy_mpl_style)
fig, axes = plt.subplots(2, 4, figsize=(20, 8))
axes = axes.flat
data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2')
obs = data_store.obs(obs_id=23523)
obs.events.peek()
obs.aeff.plot_energy_dependence(ax=next(axes))
obs.aeff.plot_offset_dependence(ax=next(axes))
aeff = obs.aeff.to_effective_area_table(offset='1 deg')
# import IPython; IPython.embed()
aeff.plot(ax=next(axes))
obs.edisp.plot_bias(ax=next(axes))
obs.edisp.plot_migration(ax=next(axes))
edisp = obs.edisp.to_energy_dispersion(offset='1 deg')
edisp.plot_matrix(ax=next(axes))
log.basicConfig(level=log.INFO)
import numpy as np
from astropy.wcs import WCS
from astropy.units import Quantity
from astropy.io import fits
from astropy.coordinates import SkyCoord
from gammapy.data import DataStore
from gammapy.cube import exposure_cube, SpectralCube
from gammapy.utils.energy import EnergyBounds
dirname = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2'
log.info('Reading data from {}'.format(dirname))
data_store = DataStore.from_dir(dirname)
events = data_store.load(obs_id=23523, filetype='events')
log.info('Number of events in event list: {}'.format(len(events)))
log.info('Max. event energy: {}'.format(events['ENERGY'].max()))
log.info('Min. event energy: {}'.format(events['ENERGY'].min()))
aeff = data_store.load(obs_id=23523, filetype='aeff')
counts = SpectralCube.empty(emin=0.5, emax=80, enbins=8, eunit='TeV',
nxpix=200, nypix=200, xref=events.meta['RA_OBJ'],
yref=events.meta['DEC_OBJ'], dtype='int',
coordsys='CEL')
log.info('Bin events into cube.')
counts.fill(events)
log.info('Counts cube shape: {}'.format(counts.data.shape))
"""
from astropy.coordinates import SkyCoord, Angle
from gammapy.datasets import gammapy_extra
from gammapy.image import ExclusionMask
from gammapy.data import DataStore
from gammapy.region import SkyCircleRegion
from gammapy.spectrum import SpectrumAnalysis
from gammapy.utils.energy import EnergyBounds
center = SkyCoord(83.63, 22.01, unit='deg', frame='icrs')
radius = Angle('0.3 deg')
on_region = SkyCircleRegion(pos=center, radius=radius)
bkg_method = dict(type='reflected', n_min=3)
exclusion_file = gammapy_extra.filename("datasets/exclusion_masks/"
"tevcat_exclusion.fits")
excl = ExclusionMask.from_fits(exclusion_file)
bounds = EnergyBounds.equal_log_spacing(1, 10, 40, unit='TeV')
store = gammapy_extra.filename("datasets/hess-crab4")
ds = DataStore.from_dir(store)
obs = [23523, 23559]
ana = SpectrumAnalysis(datastore=ds, obs=obs, on_region=on_region,
bkg_method=bkg_method, exclusion=excl, ebounds=bounds)
ana.write_ogip_data(outdir='ogip_data')
def test_data_store2():
dir = gammapy_extra.dir / 'datasets/hess-crab4-hd-hap-prod2'
ds = DataStore.from_dir(dir)
ds.info()