def test_datastore_from_dir_no_obs_index(caplog, tmpdir): """Test the `from_dir` method.""" # Create small datastore and remove obs-index table DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/").copy_obs([23523, 23592], tmpdir) os.remove(tmpdir / "obs-index.fits.gz") data_store = DataStore.from_dir(tmpdir) obs = data_store.obs(23523) observations = data_store.get_observations() assert data_store.obs_table is None assert "No observation index table." in data_store.info(show=False) assert obs.obs_info["ONTIME"] == 1687.0 assert len(observations) == 2 test_dir = tmpdir / "test" os.mkdir(test_dir) data_store.copy_obs([23523], test_dir) data_store_copy = DataStore.from_dir(test_dir) assert len(data_store_copy.obs_ids) == 1 assert data_store_copy.obs_table == None
def test_image_pipe(tmpdir): tmpdir = str(tmpdir) from subprocess import call outdir = tmpdir outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2") ds.copy_obs(ds.obs_table, tmpdir) data_store = DataStore.from_dir(tmpdir) bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True) tmpdir = str(tmpdir) center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir(tmpdir) # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) # Pb with the load psftable for one of the run that is not implemented yet... data_store.hdu_table.remove_row(14) mosaic = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) mosaic.make_images(make_background_image=True, for_integral_flux=True, radius=10.) assert_allclose(mosaic.maps['counts'].data.sum(), 2334.0, atol=3) assert_allclose(mosaic.maps['bkg'].data.sum(), 1987.1513636663785, atol=3) assert_allclose(mosaic.maps['exposure'].data.sum(), 54190569251987.68, atol=3) assert_allclose(mosaic.maps['significance'].lookup(center), 33.707901541600634, atol=3) assert_allclose(mosaic.maps['excess'].data.sum(), 346.8486363336217, atol=3)
def copy_data(): Path("data").mkdir(exist_ok=True) ds_in = DataStore.from_dir(path_in) ds_out = DataStore.from_dir(".") for obs_id in obs_ids: loc_in = ds_in.obs(obs_id).location(hdu_type="events") loc_out = ds_out.obs(obs_id).location(hdu_type="events") src = path_in / loc_in.file_dir / loc_in.file_name dst = Path(loc_out.file_dir) / loc_out.file_name print(f"cp {src} {dst}") shutil.copy(src, dst)
def test_image_pipe(tmpdir): tmpdir = str(tmpdir) from subprocess import call outdir = tmpdir outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2") ds.copy_obs(ds.obs_table, tmpdir) data_store = DataStore.from_dir(tmpdir) bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True) tmpdir = str(tmpdir) center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir(tmpdir) # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyMap.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = ExclusionMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) images = ImageAnalysis(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) images.make_maps(radius=10., bkg_norm=True, spectral_index=2.3, for_integral_flux=True)
def test_datastore_from_dir(): """Test the `from_dir` method.""" data_store_rel_path = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/", "hdu-index.fits.gz", "obs-index.fits.gz") data_store_abs_path = DataStore.from_dir( "$GAMMAPY_DATA/hess-dl3-dr1/", "$GAMMAPY_DATA/hess-dl3-dr1/hdu-index.fits.gz", "$GAMMAPY_DATA/hess-dl3-dr1/obs-index.fits.gz", ) assert "Data store" in data_store_rel_path.info(show=False) assert "Data store" in data_store_abs_path.info(show=False)
def cli_check_data2(dataset): """Check DL3 with DataStore.check Write report in results/data_checks. """ from gammapy.data import DataStore from joint_crab.utils import write_yaml path = Path("results/data_checks") path.mkdir(parents=True, exist_ok=True) if dataset == "all": names = ["magic", "hess", "fact", "veritas"] else: names = [dataset] for name in names: log.info(f"Running data checks for: {name}") data_store = DataStore.from_dir(f"data/{name}") results = data_store.check() results = (_ for _ in results if _["level"] not in {"debug", "info"}) if name in ["magic", "fact", "veritas"]: results = (_ for _ in results if _["msg"] != "Loading psf failed") results = list(results) filename = f"results/data_checks/check-data-{name}2.yaml" write_yaml(results, filename)
def create_data(input_dir, dataset_config, exclusion_map=None): telescope = dataset_config['telescope'] on_region_radius = dataset_config['on_radius'] energy_bins = dataset_config['e_reco_bins'] containment = dataset_config['containment_correction'] ds = DataStore.from_dir(os.path.join(input_dir, telescope)) observations = ds.get_observations(ds.obs_table['OBS_ID'].data) t_obs = sum([o.observation_live_time_duration for o in observations]) # from IPython import embed; embed() print(f'Total obstime for {telescope} is {t_obs.to("h")}') source_position = dataset_config['source_position'] on_region = CircleSkyRegion(center=source_position, radius=on_region_radius) print('Estimating Background') bkg_estimate = ReflectedRegionsBackgroundEstimator( observations=observations, on_region=on_region, exclusion_mask=exclusion_map) bkg_estimate.run() print('Extracting Count Spectra') extract = SpectrumExtraction( observations=observations, bkg_estimate=bkg_estimate.result, e_true=energy_bins, e_reco=energy_bins, containment_correction=containment, use_recommended_erange=False, # TODO this might have to be checked. ) extract.run() return extract
def run_test_fitsexport(directory): """Run example analysis to test a fits data production hap-data-fits-export crab has to be run in order to produce the example data """ log.info('Running test analysis of fits data') from gammapy.data import DataStore from gammapy.datasets import gammapy_extra from gammapy.utils.scripts import read_yaml from gammapy.spectrum.spectrum_pipe import run_spectrum_analysis_using_config from gammapy.spectrum.results import SpectrumResult s = DataStore.from_dir(directory) print(s.info()) configfile = gammapy_extra.filename( 'test_datasets/spectrum/spectrum_analysis_example.yaml') config = read_yaml(configfile) config['extraction']['data']['datastore'] = directory config['extraction']['data']['runlist'] = [23523, 23526, 23559, 23592] fit, analysis = run_spectrum_analysis_using_config(config) res = SpectrumResult( fit=fit.result, stats=analysis.observations.total_spectrum.spectrum_stats) print(res.to_table())
def make_counts_image(dataset, max_runs): log.info(f'Making all-sky image for dataset: {dataset}') data_store = DataStore.from_dir(f'1dc/1dc/index/{dataset}') image = SkyImage.empty( nxpix=3600, nypix=1800, binsz=0.1, xref=0, yref=0, proj='AIT', coordsys='GAL', ) # Define energy band # energy_band = Energy([1, 10], 'TeV') obs_ids = list(data_store.obs_table['OBS_ID']) obs_ids = obs_ids[:max_runs] if max_runs > 0 else obs_ids # Show a progress bar from tqdm import tqdm obs_ids = tqdm(obs_ids) for obs_id in obs_ids: events = data_store.obs(obs_id).events image.fill_events(events) image.data = image.data.astype('float32') filename = f'checks/images/allsky_counts_{dataset}.fits.gz' log.info(f'Writing {filename}') image.write(filename, overwrite=True)
def make_image(): table = Table.read('acceptance_curve.fits') table.pprint() center = SkyCoord(83.63, 22.01, unit='deg').galactic counts_image = make_empty_image(nxpix=1000, nypix=1000, binsz=0.01, xref=center.l.deg, yref=center.b.deg, proj='TAN') bkg_image = counts_image.copy() data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') for events in data_store.load_all("events"): center = events.pointing_radec.galactic livetime = events.observation_live_time_duration solid_angle = Angle(0.01, "deg") ** 2 counts_image.data += bin_events_in_image(events, counts_image).data #interp_param = dict(bounds_error=False, fill_value=None) acc_hdu = fill_acceptance_image(bkg_image.header, center, table["offset"], table["Acceptance"]) acc = Quantity(acc_hdu.data, table["Acceptance"].unit) * solid_angle * livetime bkg_image.data += acc.decompose() print(acc.decompose().sum()) counts_image.writeto("counts_image.fits", clobber=True) bkg_image.writeto("bkg_image.fits", clobber=True)
def make_image_from_2d_bg(): center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir( '/Users/jouvin/Desktop/these/temp/bg_model_image/') # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = SkyMask.read( '$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) mosaic_images = StackedObsImageMaker(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) mosaic_images.make_images(make_background_image=True, for_integral_flux=True, radius=10., make_psf=True, region_center=center) filename = 'fov_bg_images.fits' log.info('Writing {}'.format(filename)) mosaic_images.images.write(filename, clobber=True)
def make_bg_model_two_groups(): from subprocess import call outdir = '/Users/jouvin/Desktop/these/temp/bg_model_image/' outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) cmd = 'cp -r $GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/ {}'.format(outdir) print('Executing: {}'.format(cmd)) call(cmd, shell=True) data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True)
def run_benchmark(): info = {"n_obs": N_OBS} t = time.time() data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/") OBS_ID = 110380 obs_ids = OBS_ID * np.ones(N_OBS) observations = data_store.get_observations(obs_ids) info["data_loading"] = time.time() - t t = time.time() m = Map.create() for obs in observations: m.fill_events(obs.events) info["filling"] = time.time() - t t = time.time() m.write("survey_map.fits.gz", overwrite=True) info["writing"] = time.time() - t with open("bench.yaml", "w") as fh: yaml.dump(info, fh, sort_keys=False, indent=4)
def make_bg_model_two_groups(): from subprocess import call outdir = '/Users/jouvin/Desktop/these/temp/bg_model_image/' outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) cmd = 'cp -r $GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/ {}'.format( outdir) print('Executing: {}'.format(cmd)) call(cmd, shell=True) data_store = DataStore.from_dir( '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True)
def load_data(input_dir, dataset_config, exclusion_map=None): on_region_radius = dataset_config['on_radius'] e_reco_bins = dataset_config['e_reco_bins'] e_true_bins = dataset_config['e_true_bins'] containment = dataset_config['containment_correction'] ds = DataStore.from_dir(input_dir) observations = ds.get_observations(ds.obs_table['OBS_ID'].data) # observations = ds.get_observations(ds.hdu_table['OBS_ID'].data) # this is plenty wrong source_position = dataset_config['source_position'] on_region = CircleSkyRegion(center=source_position, radius=on_region_radius) print('Estimating Background') bkg_estimate = ReflectedRegionsBackgroundEstimator( observations=observations, on_region=on_region, exclusion_mask=exclusion_map) bkg_estimate.run() print('Extracting Count Spectra') extract = SpectrumExtraction( observations=observations, bkg_estimate=bkg_estimate.result, e_true=e_true_bins, e_reco=e_reco_bins, containment_correction=containment, use_recommended_erange=False, ) extract.run() if dataset_config['stack']: return [extract.spectrum_observations.stack()] else: return extract.spectrum_observations
def test_datastore_fixed_rad_max(): data_store = DataStore.from_dir("$GAMMAPY_DATA/joint-crab/dl3/magic") observations = data_store.get_observations([5029748], required_irf=['aeff', 'edisp']) assert len(observations) == 1 obs = observations[0] assert obs.rad_max is not None assert obs.rad_max.quantity.shape == (1, 1) assert u.allclose(obs.rad_max.quantity, np.sqrt(0.02) * u.deg) # test it also works with edisp (removing aeff) obs = data_store.get_observations([5029748], required_irf=['aeff', 'edisp'])[0] obs.aeff = None assert obs.rad_max is not None assert obs.rad_max.quantity.shape == (1, 1) assert u.allclose(obs.rad_max.quantity, 0.1414213 * u.deg) # removing the last irf means we have no rad_max info obs = data_store.get_observations([5029748], required_irf=['aeff', 'edisp'])[0] obs.aeff = None obs.edisp = None assert obs.rad_max is None
def make_new_directorydataset_listobs(nobs, config_directory, source_name, center, obsdir, list_obs): """ Creates a directory with only the run used for the Images of the source and create a new index table with the background aceeptance curve location to used for the bkg image. Used a list of observation not a selection at less tat 2deg from the pointing position Parameters ---------- nobs: number of observation you want config_directory: name of the config chains used to produce the data source_name: name of the source you want to compute the image center: SkyCoord of the source obsdir: directory where you want to put these data list_obs: list of obs id we want to create the new data_store Returns ------- """ ds = DataStore.from_dir(config_directory) list_ind = list() for obs in list_obs: try: i = np.where(ds.obs_table["OBS_ID"] == obs)[0][0] list_ind.append(i) except Exception: print(obs) continue try: shutil.rmtree(obsdir) except Exception: pass ds.copy_obs(ds.obs_table[list_ind], obsdir)
def make_image(): table = Table.read('acceptance_curve.fits') table.pprint() center = SkyCoord(83.63, 22.01, unit='deg').galactic counts_image = SkyMap.empty(nxpix=1000, nypix=1000, binsz=0.01, xref=center.l.deg, yref=center.b.deg, proj='TAN').to_image_hdu() bkg_image = counts_image.copy() data_store = DataStore.from_dir( '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') for events in data_store.load_all("events"): center = events.pointing_radec.galactic livetime = events.observation_live_time_duration solid_angle = Angle(0.01, "deg")**2 counts_image.data += bin_events_in_image(events, counts_image).data # interp_param = dict(bounds_error=False, fill_value=None) acc_hdu = fill_acceptance_image(bkg_image.header, center, table["offset"], table["Acceptance"]) acc = Quantity(acc_hdu.data, table["Acceptance"].unit) * solid_angle * livetime bkg_image.data += acc.decompose() print(acc.decompose().sum()) counts_image.writeto("counts_image_save.fits", clobber=True) bkg_image.writeto("bkg_image_save.fits", clobber=True)
def __init__(self, prod): self.ref_dict = prod self.ds = DataStore.from_dir(prod["datastore"]) self.ref_energy = 1 * u.TeV self.ref_offset = 0.25 * u.deg self.ref_rad = np.arange(0, 2, 0.1) * u.deg self.ref_migra = 0.95 self.observation = self.ds.obs(prod["test_obs"])
def get_DataStore(self): """get gammapy DataStore object""" if self.name in ["fermi", "joint"]: return None elif self.name in ["magic", "hess", "fact", "veritas"]: return DataStore.from_dir(self.data_path) else: raise ValueError(f"No datastore for: {self.name}")
def data_prep(): data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/") OBS_ID = 110380 obs_ids = OBS_ID * np.ones(N_OBS) observations = data_store.get_observations(obs_ids) energy_axis = MapAxis.from_bounds(0.1, 10, nbin=10, unit="TeV", name="energy", interp="log") geom = WcsGeom.create( skydir=(0, 0), binsz=0.02, width=(10, 8), frame="galactic", proj="CAR", axes=[energy_axis], ) energy_axis_true = MapAxis.from_bounds(0.05, 20, nbin=30, unit="TeV", name="energy_true", interp="log") offset_max = 4 * u.deg maker = MapDatasetMaker() safe_mask_maker = SafeMaskMaker(methods=["offset-max"], offset_max=offset_max) stacked = MapDataset.create(geom=geom, energy_axis_true=energy_axis_true) spatial_model = PointSpatialModel(lon_0="-0.05 deg", lat_0="-0.05 deg", frame="galactic") spectral_model = ExpCutoffPowerLawSpectralModel( index=2, amplitude=3e-12 * u.Unit("cm-2 s-1 TeV-1"), reference=1.0 * u.TeV, lambda_=0.1 / u.TeV, ) model = SkyModel(spatial_model=spatial_model, spectral_model=spectral_model, name="gc-source") datasets = Datasets([]) for idx, obs in enumerate(observations): cutout = stacked.cutout(obs.pointing_radec, width=2 * offset_max, name=f"dataset{idx}") dataset = maker.run(cutout, obs) dataset = safe_mask_maker.run(dataset, obs) dataset.models = [model, FoVBackgroundModel(dataset_name=dataset.name)] datasets.append(dataset) return datasets
def data_prep(): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") OBS_ID = 23523 obs_ids = OBS_ID * np.ones(N_OBS) observations = data_store.get_observations(obs_ids) target_position = SkyCoord(ra=83.63, dec=22.01, unit="deg", frame="icrs") on_region_radius = Angle("0.11 deg") on_region = CircleSkyRegion(center=target_position, radius=on_region_radius) exclusion_region = CircleSkyRegion( center=SkyCoord(183.604, -8.708, unit="deg", frame="galactic"), radius=0.5 * u.deg, ) skydir = target_position.galactic exclusion_mask = Map.create( npix=(150, 150), binsz=0.05, skydir=skydir, proj="TAN", coordsys="GAL" ) mask = exclusion_mask.geom.region_mask([exclusion_region], inside=False) exclusion_mask.data = mask e_reco = MapAxis.from_bounds(0.1, 40, nbin=40, interp="log", unit="TeV").edges e_true = MapAxis.from_bounds(0.05, 100, nbin=200, interp="log", unit="TeV").edges dataset_maker = SpectrumDatasetMaker( region=on_region, e_reco=e_reco, e_true=e_true, containment_correction=True ) bkg_maker = ReflectedRegionsBackgroundMaker(exclusion_mask=exclusion_mask) safe_mask_masker = SafeMaskMaker(methods=["aeff-max"], aeff_percent=10) spectral_model = PowerLawSpectralModel( index=2, amplitude=2e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.TeV ) spatial_model = PointSpatialModel( lon_0=target_position.ra, lat_0=target_position.dec, frame="icrs" ) spatial_model.lon_0.frozen = True spatial_model.lat_0.frozen = True sky_model = SkyModel( spatial_model=spatial_model, spectral_model=spectral_model, name="" ) # Data preparation datasets = [] for ind, observation in enumerate(observations): dataset = dataset_maker.run(observation, selection=["counts", "aeff", "edisp"]) dataset_on_off = bkg_maker.run(dataset, observation) dataset_on_off = safe_mask_masker.run(dataset_on_off, observation) dataset_on_off.name = f"dataset{ind}" dataset_on_off.models = sky_model datasets.append(dataset_on_off) return Datasets(datasets)
def get_observations(): data_store = DataStore.from_dir("$CTADATA/index/gc") t = data_store.obs_table pos_obs = SkyCoord(t["RA_PNT"], t["DEC_PNT"], unit="deg") offset = config.target_skycoord.separation(pos_obs) mask = offset < config.offset_max obs_id = t["OBS_ID"][mask] if config.debug: obs_id = obs_id[:3] return data_store.obs_list(obs_id)
def get_observations(self): """Fetch observations from the data store according to criteria defined in the configuration.""" observations_settings = self.config.observations path = make_path(observations_settings.datastore) if path.is_file(): self.datastore = DataStore.from_file(path) elif path.is_dir(): self.datastore = DataStore.from_dir(path) else: raise FileNotFoundError(f"Datastore not found: {path}") log.info("Fetching observations.") if (len(observations_settings.obs_ids) and observations_settings.obs_file is not None): raise ValueError( "Values for both parameters obs_ids and obs_file are not accepted." ) elif (not len(observations_settings.obs_ids) and observations_settings.obs_file is None): obs_list = self.datastore.get_observations() ids = [obs.obs_id for obs in obs_list] elif len(observations_settings.obs_ids): obs_list = self.datastore.get_observations( observations_settings.obs_ids) ids = [obs.obs_id for obs in obs_list] else: path = make_path(observations_settings.obs_file) ids = list( Table.read(path, format="ascii", data_start=0).columns[0]) if observations_settings.obs_cone.lon is not None: cone = dict( type="sky_circle", frame=observations_settings.obs_cone.frame, lon=observations_settings.obs_cone.lon, lat=observations_settings.obs_cone.lat, radius=observations_settings.obs_cone.radius, border="0 deg", ) selected_cone = self.datastore.obs_table.select_observations(cone) ids = list(set(ids) & set(selected_cone["OBS_ID"].tolist())) self.observations = self.datastore.get_observations(ids, skip_missing=True) if observations_settings.obs_time.start is not None: start = observations_settings.obs_time.start stop = observations_settings.obs_time.stop if len(start.shape) == 0: time_intervals = [(start, stop)] else: time_intervals = [(tstart, tstop) for tstart, tstop in zip(start, stop)] self.observations = self.observations.select_time(time_intervals) log.info(f"Number of selected observations: {len(self.observations)}") for obs in self.observations: log.debug(obs)
def make_summary_latex(): """Make summary LaTeX table for the paper.""" path = "results/summary/data.tex" log.info(f"Writing {path}") table = open(path, "w") table.write(r"\begin{tabular}{lrrrrrl}") table.write("\n") table.write(r"\hline") table.write("\n") table.write( r"Dataset & $T_{\rm obs}$ & $E_{\rm min}$ & $E_{\rm max}$ & $N_{\rm on}$ & $N_{\rm bkg}$ & $R_{\rm on}$ \\" ) table.write("\n") table.write( r" & & TeV & TeV & & & deg \\ \hline" ) table.write("\n") for name in config.all_datasets: dataset = config.datasets[name] e_min_list = [] for _spec in dataset.get_SpectrumObservationList(): e_min_list.append(_spec.lo_threshold) spec = dataset.get_SpectrumObservationList().stack() stats = spec.total_stats row_name = rf"\{name} & " if name == "fermi": T_obs = r"$\sim$7 yr & " e_min = dataset.energy_range[0].to("TeV").value E_min = f"{e_min:.2f} & " else: data_store = DataStore.from_dir(f"data/{dataset.name}") ontime = sum(data_store.obs_table["ONTIME"]) * u.s ontime = ontime.to("h").value T_obs = f"{ontime:.2f} h & " # in case of the IACT e_min is taken from the staked obs e_min = min(e_min_list) E_min = f"{e_min.to('TeV').value:.2f} & " e_max = dataset.energy_range[1].to("TeV").value E_max = f"{e_max:.0f} & " N_on = f"{stats.n_on} & " N_bkg = f"{stats.background:.1f} & " r_on = dataset.on_radius.to("deg").value R_on = rf"{r_on:.2f} \\" table.write(row_name + T_obs + E_min + E_max + N_on + N_bkg + R_on) table.write("\n") table.write("\hline") table.write("\n") table.write("\end{tabular}") table.close()
def make_counts_array(): """Make an example counts array with energy and offset axes.""" data_store = DataStore.from_dir(gammapy_extra.dir / 'datasets/hess-crab4') event_lists = data_store.load_all('events') ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = np.linspace(0, 2.5, 100) array = EnergyOffsetArray(ebounds, offset) array.fill_events(event_lists) return array
def main(): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1") obs_id = data_store.obs_table["OBS_ID"] observations = data_store.get_observations(obs_id) m = Map.create() for obs in observations: log.info(f"Processing obs_id: {obs.obs_id}") m.fill_events(obs.events) m.write("survey_map.fits.gz")
def make_counts_array(): """Make an example counts array with energy and offset axes.""" data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') event_lists = data_store.load_all('events') ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = Angle(np.linspace(0, 2.5, 100), "deg") array = EnergyOffsetArray(ebounds, offset) array.fill_events(event_lists) return array
def get_irf(self): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1") obs = data_store.obs(47828) # Using IRF for 30 deg zenith angle - can be changed later irfs = { 'aeff': obs.aeff, 'edisp': obs.edisp, 'psf': obs.psf, 'bkg': obs.bkg } return irfs
def _set_data_store(self): """Set the datastore on the Analysis object.""" path = make_path(self.config.observations.datastore) if path.is_file(): log.debug(f"Setting datastore from file: {path}") self.datastore = DataStore.from_file(path) elif path.is_dir(): log.debug(f"Setting datastore from directory: {path}") self.datastore = DataStore.from_dir(path) else: raise FileNotFoundError(f"Datastore not found: {path}")
def data_prep(): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") OBS_ID = 23523 obs_ids = OBS_ID * np.ones(N_OBS) observations = data_store.get_observations(obs_ids) target_position = SkyCoord(ra=83.63308, dec=22.01450, unit="deg") e_reco = MapAxis.from_bounds(0.1, 40, nbin=40, interp="log", unit="TeV").edges e_true = MapAxis.from_bounds(0.05, 100, nbin=200, interp="log", unit="TeV").edges on_region_radius = Angle("0.11 deg") on_region = CircleSkyRegion(center=target_position, radius=on_region_radius) dataset_maker = SpectrumDatasetMaker(containment_correction=True, selection=["counts", "aeff", "edisp"]) empty = SpectrumDatasetOnOff.create(region=on_region, e_reco=e_reco, e_true=e_true) bkg_maker = ReflectedRegionsBackgroundMaker() safe_mask_masker = SafeMaskMaker(methods=["aeff-max"], aeff_percent=10) spectral_model = PowerLawSpectralModel(index=2.6, amplitude=2.0e-11 * u.Unit("1 / (cm2 s TeV)"), reference=1 * u.TeV) spectral_model.index.frozen = False model = spectral_model.copy() model.name = "crab" datasets_1d = [] for observation in observations: dataset = dataset_maker.run(dataset=empty.copy(), observation=observation) dataset_on_off = bkg_maker.run(dataset, observation) dataset_on_off = safe_mask_masker.run(dataset_on_off, observation) datasets_1d.append(dataset_on_off) for dataset in datasets_1d: model = spectral_model.copy() model.name = "crab" dataset.model = model return datasets_1d
def test_datastore_copy_obs(tmp_path, data_store): data_store.copy_obs([23523, 23592], tmp_path, overwrite=True) substore = DataStore.from_dir(tmp_path) assert str(substore.hdu_table.base_dir) == str(tmp_path) assert len(substore.obs_table) == 2 desired = data_store.obs(23523) actual = substore.obs(23523) assert str(actual.events.table) == str(desired.events.table)
def test_fill_cube(): filename = '$GAMMAPY_EXTRA/test_datasets/background/bg_cube_model_test1.fits' array = FOVCube.read(filename, format='table', scheme='bg_cube') array.data = Quantity(np.zeros_like(array.data.value), 'u') print(type(array.data)) dir = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2' data_store = DataStore.from_dir(dir) ev_list = data_store.load_all('events') array.fill_events(ev_list) array.write('test_background.fits', format='image', clobber=True)
def test_obs(): # data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-pa') data_store.info() obs = data_store.obs(obs_id=23523) # import IPython; IPython.embed() # obs.info() # print(type(obs.events)) # print(type(obs.gti)) print(type(obs.aeff)) print(type(obs.edisp)) print(type(obs.psf))
def test_obs(): # data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-pa') data_store.info() obs = data_store.obs(obs_id=23523) # import IPython; IPython.embed() # obs.info() # print(type(obs.events)) # print(type(obs.gti)) print(type(obs.aeff)) print(type(obs.edisp)) print(type(obs.psf))
def select_data(target_position): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") selection = dict( type="sky_circle", frame="icrs", lon=target_position.ra, lat=target_position.dec, radius=2 * u.deg, ) obs_ids = data_store.obs_table.select_observations(selection)["OBS_ID"] observations = data_store.get_observations(obs_ids) return observations
def make_model(): dir = str(gammapy_extra.dir) + '/datasets/hess-crab4-hd-hap-prod2' data_store = DataStore.from_dir(dir) obs_table = data_store.obs_table ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = sqrt_space(start=0, stop=2.5, num=100) * u.deg excluded_sources = make_excluded_sources() multi_array = EnergyOffsetBackgroundModel(ebounds, offset) multi_array.fill_obs(obs_table, data_store, excluded_sources) #multi_array.fill_obs(obs_table, data_store) multi_array.compute_rate() bgarray = multi_array.bg_rate energy_range = Energy([1, 10], 'TeV') table = bgarray.acceptance_curve_in_energy_band(energy_range, energy_bins=10) multi_array.write('energy_offset_array.fits', overwrite=True) table.write('acceptance_curve.fits', overwrite=True)
def make_image_from_2d_bg(): center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir('/Users/jouvin/Desktop/these/temp/bg_model_image/') # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) mosaic_images = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) mosaic_images.make_images(make_background_image=True, for_integral_flux=True, radius=10., make_psf = True, region_center=center) filename = 'fov_bg_maps.fits' log.info('Writing {}'.format(filename)) mosaic_images.maps.write(filename, clobber=True)
def run_test_fitsexport(directory): """Run example analysis to test a fits data production hap-data-fits-export crab has to be run in order to produce the example data """ log.info('Running test analysis of fits data') from gammapy.data import DataStore from gammapy.datasets import gammapy_extra from gammapy.utils.scripts import read_yaml from gammapy.spectrum.spectrum_pipe import run_spectrum_analysis_using_config from gammapy.spectrum.results import SpectrumResult s = DataStore.from_dir(directory) print(s.info()) configfile = gammapy_extra.filename( 'test_datasets/spectrum/spectrum_analysis_example.yaml') config = read_yaml(configfile) config['extraction']['data']['datastore'] = directory config['extraction']['data']['runlist'] = [23523, 23526, 23559, 23592] fit, analysis = run_spectrum_analysis_using_config(config) res = SpectrumResult(fit=fit.result, stats=analysis.observations.total_spectrum.spectrum_stats) print(res.to_table())
def run_benchmark(): # Set up data store and select N_OBS times the observation OBS_ID data_store = DataStore.from_dir('$GAMMAPY_EXTRA/test_datasets/cta_1dc/') obs_ids = OBS_ID * np.ones(N_OBS) obs_list = data_store.obs_list(obs_id=obs_ids) target_position = SkyCoord(0, 0, unit='deg', frame='galactic') on_radius = 0.2 * u.deg on_region = CircleSkyRegion(center=target_position, radius=on_radius) bkg_estimator = RingBackgroundEstimator( r_in=0.5 * u.deg, width=0.2 * u.deg, ) # Define reference image centered on the target xref = target_position.galactic.l.value yref = target_position.galactic.b.value ref_image = SkyImage.empty( nxpix=800, nypix=600, binsz=0.02, xref=xref, yref=yref, proj='TAN', coordsys='GAL', ) exclusion_mask = ref_image.region_mask(on_region) exclusion_mask.data = 1 - exclusion_mask.data image_estimator = IACTBasicImageEstimator( reference=ref_image, emin=100 * u.GeV, emax=100 * u.TeV, offset_max=3 * u.deg, background_estimator=bkg_estimator, exclusion_mask=exclusion_mask, ) result = image_estimator.run(obs_list)
def make_reco_model(): """Make a reco bg cube model.""" METHOD = 'default' data_dir = 'test_dataset' overwrite = OVERWRITE test = TEST group_id = GROUP_ID # create output folder outdir = os.path.join(OUTDIR, 'reco') _create_dir(outdir, overwrite) # 0. create dummy observation grouping obs_groups = create_dummy_observation_grouping() # save outfile = os.path.join(outdir, 'bg_observation_groups.ecsv') print('Writing {}'.format(outfile, overwrite=overwrite)) obs_groups.write(outfile) # 1. create dummy dataset # use enough stats so that rebinning (and resmoothing?) doesn't take place n_obs = 100 if test: # run fast n_obs = 2 az_range = AZ_RANGE alt_range = ALT_RANGE random_state = np.random.RandomState(seed=0) sigma = SIGMA spectral_index = INDEX make_test_dataset(outdir=data_dir, overwrite=overwrite, observatory_name='HESS', n_obs=n_obs, az_range=az_range, alt_range=alt_range, date_range=(Time('2010-01-01T00:00:00', format='isot', scale='utc'), Time('2015-01-01T00:00:00', format='isot', scale='utc')), n_tels_range=(3, 4), sigma=sigma, spectral_index=spectral_index, random_state=random_state) # 2. get observation table data_store = DataStore.from_dir(dir=data_dir) observation_table = data_store.make_observation_table() outfile = os.path.join(outdir, 'bg_observation_table_group{}.fits.gz'.format(group_id)) print("Writing {}".format(outfile)) observation_table.write(outfile, overwrite=overwrite) # 3. build bg model method = METHOD bg_cube_model = make_bg_cube_model(observation_table=observation_table, data_dir=data_dir, method=method, do_not_force_mev_units=True) # save outfile = os.path.join(outdir, 'bg_cube_model_group{}'.format(group_id)) print("Writing {}".format('{}_table.fits.gz'.format(outfile))) print("Writing {}".format('{}_image.fits.gz'.format(outfile))) bg_cube_model.write('{}_table.fits.gz'.format(outfile), format='table', clobber=overwrite) bg_cube_model.write('{}_image.fits.gz'.format(outfile), format='image', clobber=overwrite)
def proceeding_plot(obs): # import IPython; IPython.embed() fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(12, 4)) obs.aeff.plot_energy_dependence(ax=axes[0]) edisp = obs.edisp.to_energy_dispersion(offset='1 deg') # edisp.plot_matrix(ax=axes[1]) psf_edep = obs.psf.to_table_psf(theta='1 deg') for energy in [1, 3] * u.TeV: psf = psf_edep.table_psf_at_energy(energy) psf.plot_psf_vs_theta() plt.xscale('linear') plt.yscale('linear') plt.xlim(0, 0.5) fig.tight_layout() filename = 'iact-dl3.pdf' print('Writing ', filename) fig.savefig(filename) if __name__ == '__main__': data_store = DataStore.from_dir('$HOME/code/HESS-DL3-DR1/release_store/') obs = data_store.obs(obs_id=23592) proceeding_plot(obs) peek_plots(obs)
"""Add phase for two obs on Vela pulsar from CTA DC-1""" import numpy as np import astropy.units as u from astropy.time import Time from astropy.coordinates import SkyCoord from gammapy.data import DataStore ########################################################################### ####################### Extracting Vela files ########################### ########################################################################### # Load the data store (contains the information about all the DC1 data) # Assumes you have an environment variable CTADATA set pointing to the DC1 folder data_store = DataStore.from_dir('$CTADATA/index/gps') table = data_store.obs_table # Defining the offset as the angular separation between the observation position and the target position pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg') pos_target = SkyCoord.from_name('vela') offset = pos_target.separation(pos_obs) # Defining a mask to select all runs targeting Vela with an offset < 2 deg mask = (offset < 2 * u.deg) # Applying the mask table = table[mask] # Getting the list of observation indices of Vela obs_ids = table['OBS_ID']
"""Example how to load and take a peek at an IACT observation. """ import matplotlib.pyplot as plt from gammapy.data import DataStore from gammapy.utils.mpl_style import gammapy_mpl_style # TODO: Update once this issue is resolved: # https://github.com/astropy/astropy/issues/4140 # plt.style.use(gammapy_mpl_style) plt.rcParams.update(gammapy_mpl_style) fig, axes = plt.subplots(2, 4, figsize=(20, 8)) axes = axes.flat data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') obs = data_store.obs(obs_id=23523) obs.events.peek() obs.aeff.plot_energy_dependence(ax=next(axes)) obs.aeff.plot_offset_dependence(ax=next(axes)) aeff = obs.aeff.to_effective_area_table(offset='1 deg') # import IPython; IPython.embed() aeff.plot(ax=next(axes)) obs.edisp.plot_bias(ax=next(axes)) obs.edisp.plot_migration(ax=next(axes)) edisp = obs.edisp.to_energy_dispersion(offset='1 deg') edisp.plot_matrix(ax=next(axes))
log.basicConfig(level=log.INFO) import numpy as np from astropy.wcs import WCS from astropy.units import Quantity from astropy.io import fits from astropy.coordinates import SkyCoord from gammapy.data import DataStore from gammapy.cube import exposure_cube, SpectralCube from gammapy.utils.energy import EnergyBounds dirname = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2' log.info('Reading data from {}'.format(dirname)) data_store = DataStore.from_dir(dirname) events = data_store.load(obs_id=23523, filetype='events') log.info('Number of events in event list: {}'.format(len(events))) log.info('Max. event energy: {}'.format(events['ENERGY'].max())) log.info('Min. event energy: {}'.format(events['ENERGY'].min())) aeff = data_store.load(obs_id=23523, filetype='aeff') counts = SpectralCube.empty(emin=0.5, emax=80, enbins=8, eunit='TeV', nxpix=200, nypix=200, xref=events.meta['RA_OBJ'], yref=events.meta['DEC_OBJ'], dtype='int', coordsys='CEL') log.info('Bin events into cube.') counts.fill(events) log.info('Counts cube shape: {}'.format(counts.data.shape))
""" from astropy.coordinates import SkyCoord, Angle from gammapy.datasets import gammapy_extra from gammapy.image import ExclusionMask from gammapy.data import DataStore from gammapy.region import SkyCircleRegion from gammapy.spectrum import SpectrumAnalysis from gammapy.utils.energy import EnergyBounds center = SkyCoord(83.63, 22.01, unit='deg', frame='icrs') radius = Angle('0.3 deg') on_region = SkyCircleRegion(pos=center, radius=radius) bkg_method = dict(type='reflected', n_min=3) exclusion_file = gammapy_extra.filename("datasets/exclusion_masks/" "tevcat_exclusion.fits") excl = ExclusionMask.from_fits(exclusion_file) bounds = EnergyBounds.equal_log_spacing(1, 10, 40, unit='TeV') store = gammapy_extra.filename("datasets/hess-crab4") ds = DataStore.from_dir(store) obs = [23523, 23559] ana = SpectrumAnalysis(datastore=ds, obs=obs, on_region=on_region, bkg_method=bkg_method, exclusion=excl, ebounds=bounds) ana.write_ogip_data(outdir='ogip_data')
def test_data_store2(): dir = gammapy_extra.dir / 'datasets/hess-crab4-hd-hap-prod2' ds = DataStore.from_dir(dir) ds.info()